Open source CapCut alternative.
TypeScript
fix(site): phone recordings drop the shelf badge
David Lecce1d46a6887
diff --git a/docs/guides/cut/README.md b/docs/guides/cut/README.mdindex 13e383b0..cb38f097 100644--- a/docs/guides/cut/README.md+++ b/docs/guides/cut/README.md@@ -54,6 +54,8 @@ Whether any of this is true of a given build is measured rather than assumed: `nThe machine it measures on is part of the measurement. A developer desktop has decode slots to spare and never falls behind, so the preview's caps never bind there and a change that would ruin a laptop reads as a clean sheet. `--machine` puts the page on a smaller one: it takes away hardware decode slots, so decoders past the limit deliver their frames late the way software decoding does, and it slows the main thread to a fraction of this one. `npm run eval:cut-perf-lowend` is the profile the reports come from. What the gates ask there is what a viewer would ask — did the frames arrive, did the picture hitch a quarter second behind the sound, and was the last pass as good as the first — and each machine has its own budget for those, because a number that is honest on a desktop is fantasy on a fifteen-watt laptop.+A profile it cannot model is the one someone is actually editing on, so the preview can report on itself. Settings carries a diagnostics switch, off until an account turns it on; with it on, the engine counts what its played frames did — how many arrived late, how far the picture sat behind its own sound, what the decoders and the warm shelf were holding, how long the main thread was blocked — and sends one summary per half minute of playback, alongside what the machine brought: cores, memory, the size it is decoding at, and which codecs it decodes in hardware. That last one separates the two stutters that look identical from outside: a machine spending its time after the decoder, and one whose decoder refused the footage and put every stream on the CPU. The summaries are numbers; no frame, no sound, and nothing naming the project leaves the browser.+## Color gradingA clip's color is one field on the clip: an optional preset underneath and manual adjustments on top — sliders, tone curves, color wheels, per-hue bands. Every part is absent when neutral, so an untouched clip carries nothing, and the preset's intensity scales its parameters toward neutral so half-strength means half the move, never a crossfade through gray.diff --git a/site/src/cut/components/Editor.tsx b/site/src/cut/components/Editor.tsxindex 62c8aa8c..d9fc06b4 100644--- a/site/src/cut/components/Editor.tsx+++ b/site/src/cut/components/Editor.tsx@@ -35,6 +35,8 @@ import { syncLinkedLibrary } from "@/cut/lib/linkedLibrary";import { reportActivity } from "@/cut/lib/tabActivity";import { TabStatus } from "./TabStatus";import { installDevHooks } from "@/cut/lib/devHooks";+import { meterPerf, stopMeter } from "@/cut/lib/perfTrace";+import { track } from "@/lib/analytics";import { ensureCloudThreads } from "@/cut/lib/chatCloud";import { readProjectThreads, writeActiveChat } from "@/cut/lib/chatThreads";import { backTarget, useCutBase } from "@/cut/lib/nav";@@ -90,6 +92,24 @@ export function Editor({viewer?: boolean;}) {useEffect(() => installDevHooks(), []);+ // Preview diagnostics, for accounts that turned them on in settings: the+ // engine counts what its frames did and sends a summary every half minute of+ // playback. Off — and costing a null check — for everyone else.+ useEffect(() => {+ let live = true;+ void fetch("/api/account/feature-flags")+ .then((r) => (r.ok ? (r.json() as Promise<{ flags: { id: string; enabled: boolean }[] }>) : null))+ .then((body) => {+ if (!live || !body) return;+ const on = body.flags.some((f) => f.id === "preview_diagnostics" && f.enabled);+ if (on) meterPerf((sample) => track("cut_preview_perf", sample));+ })+ .catch(() => {});+ return () => {+ live = false;+ stopMeter();+ };+ }, []);const back = backTarget(useCutBase(), from, folder);const loaded = useEditor((s) => s.loaded);const loadError = useEditor((s) => s.loadError);diff --git a/site/src/cut/components/LibraryView.tsx b/site/src/cut/components/LibraryView.tsxindex c0e898b5..a74989f8 100644--- a/site/src/cut/components/LibraryView.tsx+++ b/site/src/cut/components/LibraryView.tsx@@ -1061,7 +1061,9 @@ export function LibraryCard({const videoRef = useRef<HTMLVideoElement>(null);const { flash, attachReveal } = useRevealFlash("library", a.id);// With one shelf listed, every card is on it — the badge would say nothing.- const bothShelves = availableResidencies().length > 1 || offline;+ // Phone recordings all come up from the cloud, so they carry none either.+ const bothShelves =+ a.origin !== "camera" && (availableResidencies().length > 1 || offline);// Each card is a real media element, so the source waits until the tile has// been scrolled near: a large library would otherwise pull every file's// metadata across the network the moment the page opened.diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.tsindex 3545c46d..0434f4d8 100644--- a/site/src/cut/hooks/usePlayback.ts+++ b/site/src/cut/hooks/usePlayback.ts@@ -17,10 +17,15 @@ import { duckGainAt, overlayPlan, trackZeroPlan } from "@/cut/lib/framePlan";import { type ClipFrameSource, FrameSourcePool, mappingKey, walkCostMs } from "@/cut/lib/frameSource";import { PreviewMixer, type Voice } from "@/cut/lib/previewMixer";import {+ flushMeter,markAudioClock,markLiveSamples,markLiveSources,markPresent,+ meterAudioClock,+ meterFrame,+ meterState,+ metering,markTick,tracing,} from "@/cut/lib/perfTrace";@@ -126,6 +131,7 @@ class Engine {dispose() {engineLog(`engine ${this.serial} disposed`);+ flushMeter();this.disposed = true;if (this.raf) cancelAnimationFrame(this.raf);this.unsubscribe();@@ -209,7 +215,7 @@ class Engine {* leave what is already on screen alone. Painting black for a decoder that is* merely a moment behind is what strobing looks like.*/- private frameFor(span: ClipSpan, t: number, playing: boolean): Frame {+ private frameFor(span: ClipSpan, t: number, playing: boolean, master = false): Frame {const src = this.pool.get(keyOf(span.clip, span.asset), span.asset, this.decodeHeight());this.used.add(src);const st = sourceTimeOf(span.clip, t);@@ -221,6 +227,10 @@ class Engine {// split from one file share a source, and a held frame from across the// split would show the other scene at a paused playhead.const frame = src.frameAt(st, span.clip.in, span.clip.out);+ // The picture on track 0 is the one being watched, so it is the one the+ // machine is judged on — see the meter in `perfTrace`.+ if (master && playing && metering())+ meterFrame(frame ? Math.max(0, st - frame.timestamp) : 0, !frame);if (frame)return { kind: "ready", image: frame.image, width: frame.width, height: frame.height };return src.failed ? MISSING_FRAME : PENDING_FRAME;@@ -484,7 +494,10 @@ class Engine {// the last frame rather than clear to black.const end = Math.max(0, total - 0.001);t = Math.max(0, Math.min(previewAt(), end));- if (this.mixer.running) this.mixer.stop();+ if (this.mixer.running) {+ this.mixer.stop();+ flushMeter();+ }}// Nothing anywhere resets to a black frame at 0.@@ -511,7 +524,7 @@ class Engine {// whole design exists to remove. Leave what is on screen and stay dirty;// the clock, the audio, and the stop checks below still run, so a slow// open can't freeze the playhead or carry playback past a stop mark.- const masterFrame = master ? this.frameFor(master, t, playing) : MISSING_FRAME;+ const masterFrame = master ? this.frameFor(master, t, playing, true) : MISSING_FRAME;const pendingMaster = masterFrame.kind === "pending";const fadeGain = this.projectFadeGain(t, total);if (pendingMaster) {@@ -576,6 +589,19 @@ class Engine {markAudioClock(clock.lead, clock.reported);markLiveSources(this.pool.active, this.pool.size, this.pool.warmPixels);}++ if (playing && metering()) {+ const clock = this.mixer.clockLead();+ meterAudioClock(clock.lead, clock.reported);+ meterState({+ sources: this.pool.active,+ held: this.pool.held,+ warmMb: Math.round(this.pool.warmPixels * 4e-6),+ clips: s.clips.length,+ decodeHeight: this.decodeHeight(),+ canvasHeight: this.canvas.height,+ });+ }}}diff --git a/site/src/cut/lib/perfMeter.test.ts b/site/src/cut/lib/perfMeter.test.tsnew file mode 100644index 00000000..21026491--- /dev/null+++ b/site/src/cut/lib/perfMeter.test.ts@@ -0,0 +1,65 @@+import { describe, expect, test } from "bun:test";+import {+ flushMeter,+ meterAudioClock,+ meterFrame,+ meterPerf,+ meterState,+ metering,+ stopMeter,+ type PerfSample,+} from "./perfTrace";++describe("preview diagnostics meter", () => {+ test("a play's frames become one summary", () => {+ const out: PerfSample[] = [];+ meterPerf((s) => out.push(s));+ expect(metering()).toBe(true);+ // Three hundred frames: thirty of them a third of a second behind, ten of+ // those with no decoded picture at all.+ for (let i = 0; i < 300; i++) meterFrame(i < 30 ? 0.3 : 0, i < 10);+ meterState({+ sources: 12,+ held: 40,+ warmMb: 354,+ clips: 20,+ decodeHeight: 540,+ canvasHeight: 900,+ });+ meterAudioClock(0.16, 0.16);+ flushMeter();++ expect(out).toHaveLength(1);+ const s = out[0];+ expect(s.frames).toBe(300);+ expect(s.lateShare).toBe(0.1);+ expect(s.staleShare).toBe(0.033);+ expect(s.hitchShare).toBe(0.1);+ expect(s.lagMaxS).toBe(0.3);+ expect(s.audioLeadS).toBe(0.16);+ expect(s.audioLatencyS).toBe(0.16);+ expect(s.sources).toBe(12);+ expect(s.warmMb).toBe(354);+ expect(s.clips).toBe(20);+ expect(s.decodeHeight).toBe(540);+ stopMeter();+ expect(metering()).toBe(false);+ });++ test("a few frames are noise, and are not sent", () => {+ const out: PerfSample[] = [];+ meterPerf((s) => out.push(s));+ for (let i = 0; i < 20; i++) meterFrame(0.5, false);+ flushMeter();+ expect(out).toHaveLength(0);+ stopMeter();+ });++ test("counting costs nothing until it is armed", () => {+ expect(metering()).toBe(false);+ meterFrame(1, true);+ meterAudioClock(1, 1);+ flushMeter();+ expect(metering()).toBe(false);+ });+});diff --git a/site/src/cut/lib/perfTrace.ts b/site/src/cut/lib/perfTrace.tsindex 5e6fdab4..6ff5e370 100644--- a/site/src/cut/lib/perfTrace.ts+++ b/site/src/cut/lib/perfTrace.ts@@ -288,3 +288,292 @@ export function traceReport(t: Trace): TraceReport {longTaskMs: Math.round(Math.max(0, ...t.longTasks.map((l) => l.ms))),};}++/**+ * The preview's own report card, for the machines we cannot reproduce.+ *+ * The trace above answers a question asked in a lab: a script drives the+ * editor, reads every frame back, and holds the run to a budget. This answers+ * the same question about someone's own machine, where nobody is driving and+ * nothing can be read back — an integrated GPU with no video overlay support,+ * a Windows audio buffer several times the size of CoreAudio's, a codec the+ * hardware decoder does not take. Those preview badly in ways that are+ * invisible here, and a driver dump only narrows the guess.+ *+ * So the meter counts what the trace measures, in fixed memory: a handful of+ * running totals per played frame, flushed as one summary every half minute of+ * playback. It is off unless the account turned it on, and while it is off the+ * cost on the engine's hot path is a null check.+ *+ * Nothing here reads the project. No frame, no sound, no clip name, no file+ * name — the numbers are timings, sizes and counts of what the machine did.+ */++/** What the decoders on this machine can do in hardware. Probed once, because+ * a stutter on footage the hardware refuses is a different bug from a stutter+ * on footage it takes. */+export interface DecodeCaps {+ h264: boolean;+ hevc: boolean;+ vp9: boolean;+ av1: boolean;+}++export interface PerfSample {+ /** Which summary this is in the session, and how long the editor had been+ * open when it closed — a preview that is smooth for a minute and choppy+ * after ten is two samples that say so. */+ sample: number;+ openedForS: number;+ /** Playback this summary covers. */+ seconds: number;+ frames: number;+ fps: number;+ /** Share of frames that were not the picture belonging at that moment: the+ * stutter, as the viewer sees it. */+ lateShare: number;+ /** Share with no decoded picture at all — the preview holding a frame. */+ staleShare: number;+ /** Share more than a quarter second behind: the hitches a viewer calls out. */+ hitchShare: number;+ /** Seconds the picture sat behind the moment asked for. */+ lagMeanS: number;+ lagMaxS: number;+ /** Seconds the picture is held back for the output device, and what the+ * device says it holds. These agree when the preview is reading the clock+ * its sound leaves on; a lead of zero against a device holding a fifth of a+ * second is a picture running that far ahead of its own audio. */+ audioLeadS: number;+ audioLatencyS: number;+ /** What the preview was holding open at its peak. */+ sources: number;+ heldFrames: number;+ warmMb: number;+ /** Main thread blocked: the worst one, and how many there were. */+ longTaskMs: number;+ longTasks: number;+ /** The shape of the cut and the size of the picture, since both decide how+ * much work a frame is. */+ clips: number;+ decodeHeight: number;+ canvasHeight: number;+ dpr: number;+ /** What the machine brought. */+ cores: number;+ memoryGb: number;+ hwH264: boolean;+ hwHevc: boolean;+ hwVp9: boolean;+ hwAv1: boolean;+}++/** Playback frames in one summary: about half a minute at sixty a second. */+const SAMPLE_FRAMES = 1800;+/** Fewest frames worth reporting when a play ends early. */+const SAMPLE_MIN_FRAMES = 240;+/** Summaries one editor session will send. A day-long session is interesting+ * for its first few, and this is not a firehose. */+const SAMPLE_CAP = 40;+/** A frame this far behind is a hitch a person would name. */+const HITCH_S = 0.25;++interface Meter {+ frames: number;+ late: number;+ stale: number;+ hitches: number;+ lagSum: number;+ lagMax: number;+ audioLead: number;+ audioLatency: number;+ sources: number;+ held: number;+ warmMb: number;+ longTaskMs: number;+ longTasks: number;+ clips: number;+ decodeHeight: number;+ canvasHeight: number;+ startedAt: number;+}++let meter: Meter | null = null;+let sink: ((sample: PerfSample) => void) | null = null;+let meterObserver: PerformanceObserver | null = null;+let sampleCount = 0;+let openedAt = 0;+let caps: DecodeCaps = { h264: false, hevc: false, vp9: false, av1: false };++const emptyMeter = (): Meter => ({+ frames: 0,+ late: 0,+ stale: 0,+ hitches: 0,+ lagSum: 0,+ lagMax: 0,+ audioLead: 0,+ audioLatency: 0,+ sources: 0,+ held: 0,+ warmMb: 0,+ longTaskMs: 0,+ longTasks: 0,+ clips: 0,+ decodeHeight: 0,+ canvasHeight: 0,+ startedAt: performance.now(),+});++/** Whether anything is counting. The engine's hot path checks this first. */+export const metering = (): boolean => meter !== null;++/**+ * Start counting, and send each summary to `report`.+ *+ * Calling again replaces the sink and leaves the counts alone, so a flag read+ * that resolves twice does not restart the session's numbering.+ */+export function meterPerf(report: (sample: PerfSample) => void): void {+ sink = report;+ if (meter) return;+ openedAt = performance.now();+ meter = emptyMeter();+ void probeDecodeCaps();+ if (typeof PerformanceObserver === "undefined") return;+ try {+ meterObserver = new PerformanceObserver((list) => {+ for (const entry of list.getEntries()) {+ if (!meter) return;+ meter.longTasks++;+ meter.longTaskMs = Math.max(meter.longTaskMs, Math.round(entry.duration));+ }+ });+ meterObserver.observe({ entryTypes: ["longtask"] });+ } catch {+ meterObserver = null;+ }+}++/** Stop counting and forget the sink. */+export function stopMeter(): void {+ meterObserver?.disconnect();+ meterObserver = null;+ meter = null;+ sink = null;+}++/** One played frame: how far behind the moment asked for the picture was, and+ * whether there was a decoded picture for it at all. */+export function meterFrame(lagS: number, stale: boolean): void {+ if (!meter) return;+ meter.frames++;+ if (stale) meter.stale++;+ if (stale || lagS > SAME_TIME) meter.late++;+ if (lagS > HITCH_S) meter.hitches++;+ meter.lagSum += lagS;+ meter.lagMax = Math.max(meter.lagMax, lagS);+ if (meter.frames >= SAMPLE_FRAMES) flushMeter();+}++/** What the preview is holding open, and the size it is working at. */+export function meterState(state: {+ sources: number;+ held: number;+ warmMb: number;+ clips: number;+ decodeHeight: number;+ canvasHeight: number;+}): void {+ if (!meter) return;+ meter.sources = Math.max(meter.sources, state.sources);+ meter.held = Math.max(meter.held, state.held);+ meter.warmMb = Math.max(meter.warmMb, state.warmMb);+ meter.clips = Math.max(meter.clips, state.clips);+ meter.decodeHeight = state.decodeHeight;+ meter.canvasHeight = state.canvasHeight;+}++/** What the preview's clock is doing about the output device. */+export function meterAudioClock(lead: number, reported: number): void {+ if (!meter) return;+ meter.audioLead = Math.max(meter.audioLead, lead);+ meter.audioLatency = Math.max(meter.audioLatency, reported);+}++/**+ * Send what has been counted and start a fresh count.+ *+ * A play that ends before the window fills still reports, because a short play+ * that stuttered is the whole complaint; a handful of frames is noise and is+ * dropped instead.+ */+export function flushMeter(): void {+ const m = meter;+ if (!m || !sink) return;+ meter = emptyMeter();+ if (m.frames < SAMPLE_MIN_FRAMES || sampleCount >= SAMPLE_CAP) return;+ const seconds = (performance.now() - m.startedAt) / 1000;+ const round = (n: number, places = 3) => +n.toFixed(places);+ sink({+ sample: ++sampleCount,+ openedForS: Math.round((performance.now() - openedAt) / 1000),+ seconds: round(seconds, 1),+ frames: m.frames,+ fps: seconds > 0 ? round(m.frames / seconds, 1) : 0,+ lateShare: round(m.late / m.frames),+ staleShare: round(m.stale / m.frames),+ hitchShare: round(m.hitches / m.frames),+ lagMeanS: round(m.lagSum / m.frames),+ lagMaxS: round(m.lagMax),+ audioLeadS: round(m.audioLead),+ audioLatencyS: round(m.audioLatency),+ sources: m.sources,+ heldFrames: m.held,+ warmMb: m.warmMb,+ longTaskMs: m.longTaskMs,+ longTasks: m.longTasks,+ clips: m.clips,+ decodeHeight: m.decodeHeight,+ canvasHeight: m.canvasHeight,+ dpr: typeof devicePixelRatio === "number" ? round(devicePixelRatio, 2) : 1,+ cores: navigator.hardwareConcurrency || 0,+ memoryGb: (navigator as unknown as { deviceMemory?: number }).deviceMemory ?? 0,+ hwH264: caps.h264,+ hwHevc: caps.hevc,+ hwVp9: caps.vp9,+ hwAv1: caps.av1,+ });+}++/**+ * Which codecs this machine decodes in hardware, at 1080p.+ *+ * The answer separates two very different stutters. A machine that takes the+ * footage in hardware and still drops frames is spending its time somewhere+ * after the decoder — converting frames, uploading them, compositing. A+ * machine whose decoder refuses the codec is running every stream on the CPU,+ * where a fast cut has no chance. Both look identical from the outside.+ */+async function probeDecodeCaps(): Promise<void> {+ if (typeof VideoDecoder === "undefined") return;+ const ask = async (codec: string): Promise<boolean> => {+ try {+ const support = await VideoDecoder.isConfigSupported({+ codec,+ codedWidth: 1920,+ codedHeight: 1080,+ hardwareAcceleration: "prefer-hardware",+ });+ return support.supported === true;+ } catch {+ return false;+ }+ };+ const [h264, hevc, vp9, av1] = await Promise.all([+ ask("avc1.640028"),+ ask("hvc1.1.6.L120.90"),+ ask("vp09.00.41.08"),+ ask("av01.0.09M.08"),+ ]);+ caps = { h264, hevc, vp9, av1 };+}diff --git a/site/src/lib/analytics.ts b/site/src/lib/analytics.tsindex 2f67f55b..b0a5f769 100644--- a/site/src/lib/analytics.ts+++ b/site/src/lib/analytics.ts@@ -1,6 +1,7 @@import posthog from "posthog-js";import { useEffect } from "react";+import type { PerfSample } from "@/cut/lib/perfTrace";import type {OnboardingRun,ReferralSource,@@ -28,6 +29,10 @@ export type AnalyticsEvents = {cut_storage_pill_clicked: void;cut_storage_upgrade_shown: { source: "quota-413" | "pill" };cut_grace_banner_shown: void;+ // Preview timing from an account that turned diagnostics on, one summary per+ // half minute of playback. Numbers only — see `PerfSample` in the Cut+ // preview's perfTrace for what each one means and why it is here.+ cut_preview_perf: PerfSample;// Billing (settings).pro_checkout_started: void;billing_portal_opened: void;diff --git a/site/src/lib/feature-flags.ts b/site/src/lib/feature-flags.tsindex ebf175ea..1cf02bbb 100644--- a/site/src/lib/feature-flags.ts+++ b/site/src/lib/feature-flags.ts@@ -10,7 +10,15 @@ export type AccountFeatureFlag = {defaultEnabled: boolean;};-export const ACCOUNT_FEATURE_FLAGS: AccountFeatureFlag[] = [];+export const ACCOUNT_FEATURE_FLAGS: AccountFeatureFlag[] = [+ {+ id: "preview_diagnostics",+ title: "Share preview diagnostics",+ description:+ "Sends timing from the preview while you play: frames that arrived late, how far the picture sat behind the sound, and what your machine was holding open. It is how we fix stutters on machines we cannot reproduce. Your video, audio and project stay in your browser.",+ defaultEnabled: false,+ },+];export const isKnownFeatureFlag = (id: string) =>ACCOUNT_FEATURE_FLAGS.some((f) => f.id === id);