Skip to content

Open source CapCut alternative.

TypeScript

feat(site): a link's sound can come down alone [rebuild]

David Le33374d01e49e
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.ts
index 3095eeda..7fb127ac 100644
--- a/site/src/cut/components/AiPanel.tools.ts
+++ b/site/src/cut/components/AiPanel.tools.ts
@@ -97,8 +97,16 @@ export const AI_PANEL_TOOLS = [
{
name: "import_url",
description:
- "Read any URL — TikTok, YouTube, Instagram Reels, an X/Twitter post or Article, an ordinary web page, or a direct video/audio/image link — with the bundled downloader and import what it holds into the project. Free and local. A web page comes back as its article text plus the pictures on it; a post as its video or photos; and the source's own words (returned as sourceText) are quoted for the user beside the media automatically — don't retype them in your reply. A source that is only words returns sourceText with no assets, which is a success: read it and answer from it. This is how you look something up: point it at the page and read what comes back. Media lands on a card in this chat and the user drags it to the timeline, Media, or the Library; place it yourself (add_clip) only when they asked for it in the cut. A short clip downloads in seconds; a long video can take a couple of minutes.",
- inputSchema: obj({ url: str("The page or media URL to download") }, ["url"]),
+ "Read any URL — TikTok, YouTube, Instagram Reels, an X/Twitter post or Article, an ordinary web page, or a direct video/audio/image link — with the bundled downloader and import what it holds into the project. Free and local. A web page comes back as its article text plus the pictures on it; a post as its video or photos; and the source's own words (returned as sourceText) are quoted for the user beside the media automatically — don't retype them in your reply. A source that is only words returns sourceText with no assets, which is a success: read it and answer from it. This is how you look something up: point it at the page and read what comes back. When the user wants only the sound — a song, a soundtrack, a podcast — pass audio_only and the source's audio track lands as an audio asset. Media lands on a card in this chat and the user drags it to the timeline, Media, or the Library; place it yourself (add_clip) only when they asked for it in the cut. A short clip downloads in seconds; a long video can take a couple of minutes.",
+ inputSchema: obj(
+ {
+ url: str("The page or media URL to download"),
+ audio_only: bool(
+ "Import only the source's audio track, extracted to an audio file (default false)"
+ ),
+ },
+ ["url"]
+ ),
},
{
name: "list_skills",
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 074603b6..09436fd0 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -2563,7 +2563,9 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
// Captured before the download: the media files under the chat that
// asked, even if the user switches threads while it downloads.
const chatId = chatOwner();
- const { assets, text } = await importUrlMedia(projectId, url);
+ const { assets, text } = await importUrlMedia(projectId, url, {
+ audio: input.audio_only === true,
+ });
for (const asset of assets) tagChatAsset(asset.id, chatId);
return {
assets: assets.map((asset) => ({
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 1849b2b9..caccf68f 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -845,7 +845,8 @@ export async function importStockMusic(
* an article, or a video's title and description. */
export async function importUrlMedia(
projectId: string,
- url: string
+ url: string,
+ opts: { audio?: boolean } = {}
): Promise<{ assets: MediaAsset[]; text?: string }> {
url = normalizeLink(url);
// Pinned at start: the download can outlast navigation into a project of
@@ -854,13 +855,13 @@ export async function importUrlMedia(
// A browser project has no server of its own to run the fetch, so its import
// rides the cloud worker and the bytes come back down.
if (backend.kind === "browser") {
- const staged = await importUrlThroughCloud(projectId, backend, url);
+ const staged = await importUrlThroughCloud(projectId, backend, url, opts);
return adoptImportedFiles(projectId, backend, staged);
}
const res = await backend.fetch(`/api/cut/projects/${projectId}/import-url`, {
method: "POST",
headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ url }),
+ body: JSON.stringify({ url, ...(opts.audio ? { audio: true } : {}) }),
});
let body: { files?: { fileName: string; title: string }[]; text?: string; error?: string };
if (backend.kind === "cloud") {
@@ -911,12 +912,13 @@ async function adoptImportedFiles(
async function importUrlThroughCloud(
projectId: string,
backend: CutBackend,
- url: string
+ url: string,
+ opts: { audio?: boolean } = {}
): Promise<{ files: { fileName: string; title: string }[]; text?: string }> {
const res = await cloudBackend.fetch("/api/cut/library/import-url", {
method: "POST",
headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ url }),
+ body: JSON.stringify({ url, ...(opts.audio ? { audio: true } : {}) }),
});
const started = await apiJson<{ jobId?: string }>(res);
if (!res.ok || !started.jobId) throw new Error(started.error ?? "Could not import that URL.");
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index 6408da73..12b8a7e3 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -364,7 +364,7 @@ The Library is shared across every project: folders, reusable assets, and templa
- Assets from the user's iOS app carry an origin: "camera" is a clip they recorded on their phone (their Camera Roll — "the video I shot on my phone" lives here), "inspiration" a reference saved into the Library's Inspiration folder. library_add imports either like any other asset. A camera clip's name is the clock it was shot on; its title, when it has one, was read off the clip itself and is what the user sees on the card — match a request like "the one about the launch" against the title.
- notes_list reads the user's synced notes (written on their phone or the Notes tab) with the folders they are filed in and the labels they carry — the place to look when they point at "my note", or name a folder or label, for a script, caption, or voiceover text.
- library_organize handles folders (create/rename/delete), filing (move_asset), and deletes (permanent — explicit ask only).
-- import_url reads any URL (TikTok, YouTube, Instagram, an X post or Article, a web page, direct links): a page comes back as its article text and its pictures, a post as its video or photos, and a source that is only words as sourceText alone. Media lands as a card in the chat with that text quoted beside it; the user drags a card to the timeline, Media, or the Library. Place it with add_clip only when they asked for it in the cut.
+- import_url reads any URL (TikTok, YouTube, Instagram, an X post or Article, a web page, direct links): a page comes back as its article text and its pictures, a post as its video or photos, and a source that is only words as sourceText alone. When the user asks for the sound alone — a song, a soundtrack — pass audio_only and the audio track lands as an audio asset. Media lands as a card in the chat with that text quoted beside it; the user drags a card to the timeline, Media, or the Library. Place it with add_clip only when they asked for it in the cut.
Attachments: media files dropped on the chat import into project media by themselves; library attachments wait for library_add.`,
"publish-and-export": `# Publish & export
diff --git a/site/src/cut/server/cloud/jobs.ts b/site/src/cut/server/cloud/jobs.ts
index 89ef4160..263390be 100644
--- a/site/src/cut/server/cloud/jobs.ts
+++ b/site/src/cut/server/cloud/jobs.ts
@@ -471,7 +471,7 @@ export const jobsCloud = {
* engine's synchronous route. */
async importUrl(userId: string, projectId: string, req: Request) {
try {
- const { url } = (await req.json()) as { url?: string };
+ const { url, audio } = (await req.json()) as { url?: string; audio?: boolean };
if (!url) return err("No URL provided.", 400);
if (!(await getProject(userId, projectId))) return err("Project not found.", 404);
const capped = await renderJobCheck(userId);
@@ -481,7 +481,7 @@ export const jobsCloud = {
userId,
projectId,
kind: "import_url",
- spec: { url } as unknown as Prisma.InputJsonValue,
+ spec: { url, ...(audio === true ? { audio: true } : {}) } as unknown as Prisma.InputJsonValue,
},
});
wakeRenderWorker();
@@ -496,7 +496,11 @@ export const jobsCloud = {
* the assets, and the client polls this job for them. */
async importUrlToLibrary(userId: string, req: Request) {
try {
- const { url, origin } = (await req.json()) as { url?: string; origin?: string };
+ const { url, origin, audio } = (await req.json()) as {
+ url?: string;
+ origin?: string;
+ audio?: boolean;
+ };
if (!url) return err("No URL provided.", 400);
const capped = await renderJobCheck(userId);
if (capped) return capped;
@@ -507,6 +511,7 @@ export const jobsCloud = {
spec: {
url,
target: "library",
+ ...(audio === true ? { audio: true } : {}),
// An inspiration link's downloads land in the Inspiration folder,
// carrying the origin the Camera Roll and Library filters read.
...(origin === "inspiration" ? { origin } : {}),
diff --git a/site/src/cut/server/http/library.ts b/site/src/cut/server/http/library.ts
index 1b14dc93..29b7960d 100644
--- a/site/src/cut/server/http/library.ts
+++ b/site/src/cut/server/http/library.ts
@@ -113,9 +113,9 @@ export const libraryApi = {
/** Download a media URL straight into the library. */
async importUrl(req: Request) {
try {
- const { url } = (await req.json()) as { url?: string };
+ const { url, audio } = (await req.json()) as { url?: string; audio?: boolean };
if (!url) return err("No URL provided.", 400);
- return Response.json(await importFromUrl(url));
+ return Response.json(await importFromUrl(url, { audio: audio === true }));
} catch (e) {
return caught(e, "Could not import that URL.");
}
diff --git a/site/src/cut/server/http/projects.ts b/site/src/cut/server/http/projects.ts
index 0e6724e2..8f75c1a7 100644
--- a/site/src/cut/server/http/projects.ts
+++ b/site/src/cut/server/http/projects.ts
@@ -188,9 +188,9 @@ export const projectsApi = {
/** Download a media URL straight into the project's media folder. */
async importUrl(req: Request, { id }: { id: string }) {
try {
- const { url } = (await req.json()) as { url?: string };
+ const { url, audio } = (await req.json()) as { url?: string; audio?: boolean };
if (!url) return err("No URL provided.", 400);
- return Response.json(await importUrlToProject(id, url));
+ return Response.json(await importUrlToProject(id, url, { audio: audio === true }));
} catch (e) {
return caught(e, "Could not import that URL.");
}
diff --git a/site/src/cut/server/urlDownload.ts b/site/src/cut/server/urlDownload.ts
index 14517c72..74a7ae1e 100644
--- a/site/src/cut/server/urlDownload.ts
+++ b/site/src/cut/server/urlDownload.ts
@@ -39,6 +39,13 @@ export interface Downloaded {
text?: string;
}
+export interface DownloadOptions {
+ /** Bring back only the sound: the source's audio track, landed as an audio
+ * file. Reads the media rung; a post's photos and a page's text come back as
+ * themselves. */
+ audio?: boolean;
+}
+
// A video's own words for the chat: its title and description joined, capped so
// a long YouTube description (link dumps, chapter lists) can't flood the chat.
// TikTok/Instagram often repeat the caption as both fields, so drop a
@@ -71,7 +78,11 @@ const X_URL_RE = /^https?:\/\/(?:www\.|mobile\.)?(?:x|twitter)\.com\//i;
* then the post's words alone. Every other URL tries yt-dlp, then the page
* itself — what it says and what it shows. When no rung produces anything, the
* first failure is what surfaces — it names the real cause. */
-export async function download(url: string, tmp: string): Promise<Downloaded> {
+export async function download(
+ url: string,
+ tmp: string,
+ opts: DownloadOptions = {}
+): Promise<Downloaded> {
if (IMAGE_URL_RE.test(url)) return downloadDirectImage(url, tmp);
const rungs: (() => Promise<Downloaded | null>)[] = [];
const postId = X_STATUS_RE.exec(url)?.[1];
@@ -85,14 +96,14 @@ export async function download(url: string, tmp: string): Promise<Downloaded> {
const post = fetchPost(postId);
rungs.push(
() => xPostMedia(post, url, tmp, { deferVideo: true }),
- () => downloadMedia(url, tmp),
+ () => downloadMedia(url, tmp, opts),
() => xPostMedia(post, url, tmp, { deferVideo: false }),
() => xPostText(post, url)
);
} else if (X_ARTICLE_RE.test(url)) {
rungs.push(() => xArticle({}, url, tmp));
} else {
- rungs.push(() => downloadMedia(url, tmp), () => readPage(url, tmp));
+ rungs.push(() => downloadMedia(url, tmp, opts), () => readPage(url, tmp));
}
const failures: unknown[] = [];
for (const rung of rungs) {
@@ -116,7 +127,7 @@ class MediaHostError extends Error {}
* is the last. */
class MissingToolError extends Error {}
-async function downloadMedia(url: string, tmp: string): Promise<Downloaded> {
+async function downloadMedia(url: string, tmp: string, opts: DownloadOptions = {}): Promise<Downloaded> {
// Read what sits at the URL before pulling any bytes. The extractor name
// says whether this is a site that serves video, and that decides what a
// failed download means: on a video host the import stops with the real
@@ -139,7 +150,7 @@ async function downloadMedia(url: string, tmp: string): Promise<Downloaded> {
let meta: YtMeta;
let out: string;
try {
- ({ meta, dir: out } = await runYtDlp(url, tmp, bin));
+ ({ meta, dir: out } = await runYtDlp(url, tmp, bin, opts));
} catch (e) {
if (!videoHost) throw e;
throw new MediaHostError(e instanceof Error ? e.message : "Download failed.");
@@ -710,7 +721,8 @@ const DOWNLOAD_BUDGET_MS = 8 * 60_000;
async function runYtDlp(
url: string,
dir: string,
- preferred: string
+ preferred: string,
+ opts: DownloadOptions = {}
): Promise<{ meta: YtMeta; dir: string }> {
const bins = [preferred, ...ytDlpBinaries().filter((b) => b !== preferred)];
const deadline = Date.now() + DOWNLOAD_BUDGET_MS;
@@ -722,7 +734,7 @@ async function runYtDlp(
const out = path.join(dir, `try${i}`);
await mkdir(out, { recursive: true });
try {
- return { meta: await ytDlp(bin, url, out, client, left), dir: out };
+ return { meta: await ytDlp(bin, url, out, client, left, opts), dir: out };
} catch (e) {
if (e instanceof MissingToolError) throw e;
first ??= e;
@@ -736,26 +748,40 @@ function ytDlp(
url: string,
dir: string,
client: string | null,
- timeoutMs: number
+ timeoutMs: number,
+ opts: DownloadOptions = {}
): Promise<YtMeta> {
+ // An audio import takes the best sound stream alone and lands it as m4a.
+ // AAC leads the sort — it decodes everywhere the import goes — and -x
+ // extracts the sound when the site only serves a combined file, transcoding
+ // an Opus-only stream through the bundled ffmpeg.
+ const format = opts.audio
+ ? [
+ "-f", "bestaudio/best",
+ "-S", "acodec:aac",
+ "-x", "--audio-format", "m4a",
+ ]
+ : [
+ // Take the best video and the best audio and put them together. Asking for
+ // a ready-made mp4 first would land whatever single file the site keeps
+ // around for compatibility — on YouTube that is 360p next to a 4K original.
+ //
+ // Codec leads the sort, resolution follows it. Apple decodes H.264 and
+ // HEVC and nothing else, so a taller VP9 or AV1 stream is a file that
+ // plays nowhere the import is going: the phone's viewer, the Mac's own
+ // player, Safari. Instagram serves 1440p VP9 beside a 1080p-class H.264
+ // file, and the H.264 one is worth having. Sound is compared last, after
+ // the picture: judged any earlier it would rank YouTube's 360p
+ // sound-and-picture file over the 1080p stream that carries no sound of
+ // its own. A source with no H.264 at all still comes down, and the
+ // conversion below makes it playable.
+ "-f", "bestvideo*+bestaudio/best",
+ "-S", "vcodec:h264,res,fps,hdr:sdr,acodec:aac",
+ "--merge-output-format", "mp4",
+ ];
return ytDlpJson(bin, [
...(client ? ["--extractor-args", `youtube:player_client=${client}`] : []),
- // Take the best video and the best audio and put them together. Asking for
- // a ready-made mp4 first would land whatever single file the site keeps
- // around for compatibility — on YouTube that is 360p next to a 4K original.
- //
- // Codec leads the sort, resolution follows it. Apple decodes H.264 and
- // HEVC and nothing else, so a taller VP9 or AV1 stream is a file that
- // plays nowhere the import is going: the phone's viewer, the Mac's own
- // player, Safari. Instagram serves 1440p VP9 beside a 1080p-class H.264
- // file, and the H.264 one is worth having. Sound is compared last, after
- // the picture: judged any earlier it would rank YouTube's 360p
- // sound-and-picture file over the 1080p stream that carries no sound of
- // its own. A source with no H.264 at all still comes down, and the
- // conversion below makes it playable.
- "-f", "bestvideo*+bestaudio/best",
- "-S", "vcodec:h264,res,fps,hdr:sdr,acodec:aac",
- "--merge-output-format", "mp4",
+ ...format,
"-o", path.join(dir, "%(id)s.%(ext)s"),
"--print-json",
url,
diff --git a/site/src/cut/server/urlImport.ts b/site/src/cut/server/urlImport.ts
index b344a0d2..823a623b 100644
--- a/site/src/cut/server/urlImport.ts
+++ b/site/src/cut/server/urlImport.ts
@@ -4,7 +4,7 @@ import path from "node:path";
import { addDownloaded, type LibraryAsset, type LibrarySource } from "./library";
import { assertLocalRuntime } from "./local-only";
import { mediaDir, mediaPath as projectMediaPath, readProject } from "./projects";
-import { download, type Downloaded } from "./urlDownload";
+import { download, type Downloaded, type DownloadOptions } from "./urlDownload";
import { uniqueName } from "./util";
// Import a URL (TikTok, YouTube, Instagram, an X post, a page, …) into the
@@ -21,7 +21,11 @@ let active = 0;
* inside the try so a failure there still releases the slot (else a rejection
* would leak it and, after MAX_ACTIVE failures, brick URL import until
* restart). */
-async function withDownload<T>(url: string, consume: (dl: Downloaded) => Promise<T>): Promise<T> {
+async function withDownload<T>(
+ url: string,
+ opts: DownloadOptions,
+ consume: (dl: Downloaded) => Promise<T>
+): Promise<T> {
assertLocalRuntime();
if (!/^https?:\/\//i.test(url.trim())) throw new Error("Enter a valid http(s) URL.");
if (active >= MAX_ACTIVE) {
@@ -31,7 +35,7 @@ async function withDownload<T>(url: string, consume: (dl: Downloaded) => Promise
try {
const tmp = await mkdtemp(path.join(os.tmpdir(), "cut-dl-"));
try {
- return await consume(await download(url.trim(), tmp));
+ return await consume(await download(url.trim(), tmp, opts));
} finally {
void rm(tmp, { recursive: true, force: true });
}
@@ -44,8 +48,11 @@ async function withDownload<T>(url: string, consume: (dl: Downloaded) => Promise
* A multi-photo post lands as one asset per photo. The Library holds media, so
* a source that turned out to be only words fails here — the chat's import is
* where text lands. */
-export async function importFromUrl(url: string): Promise<LibraryAsset[]> {
- return withDownload(url, async (dl) => {
+export async function importFromUrl(
+ url: string,
+ opts: DownloadOptions = {}
+): Promise<LibraryAsset[]> {
+ return withDownload(url, opts, async (dl) => {
if (dl.files.length === 0) throw new Error("That link has no media to import.");
const assets: LibraryAsset[] = [];
for (const f of dl.files) {
@@ -61,10 +68,11 @@ export async function importFromUrl(url: string): Promise<LibraryAsset[]> {
* the assets. */
export async function importUrlToProject(
projectId: string,
- url: string
+ url: string,
+ opts: DownloadOptions = {}
): Promise<{ files: { fileName: string; title: string }[]; source: LibrarySource; text?: string }> {
if (!(await readProject(projectId))) throw new Error("Project not found.");
- return withDownload(url, async (dl) => {
+ return withDownload(url, opts, async (dl) => {
await mkdir(mediaDir(projectId), { recursive: true });
const files: { fileName: string; title: string }[] = [];
for (const f of dl.files) {
diff --git a/site/src/cut/worker/importUrlJob.ts b/site/src/cut/worker/importUrlJob.ts
index 7a903eaa..9d062b49 100644
--- a/site/src/cut/worker/importUrlJob.ts
+++ b/site/src/cut/worker/importUrlJob.ts
@@ -35,10 +35,11 @@ export async function runImportUrlJob(
job: ClaimedJob,
isCanceled: () => boolean
): Promise<ImportUrlResult | LibraryImportResult> {
- const { url, target, origin } = (job.spec ?? {}) as {
+ const { url, target, origin, audio } = (job.spec ?? {}) as {
url?: string;
target?: string;
origin?: string;
+ audio?: boolean;
};
if (!url || !/^https?:\/\//i.test(url.trim())) throw new Error("Enter a valid http(s) URL.");
const toLibrary = target === "library";
@@ -59,7 +60,7 @@ export async function runImportUrlJob(
const tmp = await mkdtemp(path.join(os.tmpdir(), "cut-dl-"));
try {
- const dl = await download(url.trim(), tmp);
+ const dl = await download(url.trim(), tmp, { audio: audio === true });
if (isCanceled()) throw new Error("Import canceled.");
const rows = await prisma.cutMediaObject.findMany({