Skip to content

🎥 Make videos programmatically with React

Merge remote-tracking branch 'origin/main' into codex/canvas-capture-shot-tools

JonnyBurgerb9a38592609e
diff --cc packages/canvas-capture-extension/src/content.ts
index 68e0b1a20f,470bec6217..7c9ea93730
--- a/packages/canvas-capture-extension/src/content.ts
+++ b/packages/canvas-capture-extension/src/content.ts
@@@ -1031,6 -990,151 +999,170 @@@ export const startContent = () =>
controlsBusy || state.recording || startingRecording;
};
+ const refreshEncoderSupport = async () => {
+ const checkId = ++encoderSupportCheckId;
+ if (
+ !supported ||
+ !selectedTarget ||
+ capture ||
+ completedRecording ||
+ finalizing ||
+ selecting ||
+ selectionDrag
+ ) {
+ if (!selectedTarget) {
+ encoderSupport = 'unavailable';
+ encoderSupportKey = null;
+ outputSize = null;
+ }
+
+ return;
+ }
+
+ const target = resolveCaptureTarget();
+ if (!target) {
+ encoderSupport = 'unavailable';
+ encoderSupportKey = null;
+ outputSize = null;
+ return;
+ }
+
+ try {
+ const source = getCapturePreflight({scale: 0.1, crop: target.crop});
+ const selectedSize = target.crop ?? source.sourceSize;
+ const targetKey = `${selectedSize.width}x${selectedSize.height}`;
+ const zoomLimitKey = `${targetKey}:${source.sourceSize.width}x${source.sourceSize.height}:${window.devicePixelRatio}`;
+ if (zoomLimit?.key !== zoomLimitKey) {
+ zoomLimit = {
+ key: zoomLimitKey,
+ targetKey,
+ maximum: getSuggestedMaximumCaptureScale(
+ selectedSize.width,
+ selectedSize.height,
+ ),
+ pending: getMaximumCaptureScale(target.crop),
+ };
+ encoderSupport = 'checking';
+ encoderSupportKey = null;
+ updateControls();
+ }
+
+ const currentZoomLimit = zoomLimit;
+ const maximumScale = await currentZoomLimit.pending.catch((error) => {
+ if (zoomLimit === currentZoomLimit) {
+ zoomLimit = null;
+ }
+
+ throw error;
+ });
+ if (checkId !== encoderSupportCheckId) {
+ return;
+ }
+
+ currentZoomLimit.maximum = maximumScale ?? 0.1;
+ if (useDefaultResolution) {
+ scale = getDefaultResolutionScale(
+ selectedSize.width,
+ selectedSize.height,
+ );
+ }
+
+ scale = Math.min(scale, currentZoomLimit.maximum);
+ updateHighlight();
+ if (maximumScale === null) {
+ throw new Error('No supported recording resolution for this area.');
+ }
+
+ const preflight = getCapturePreflight({
+ scale,
+ crop: target.crop,
+ });
+ if (checkId !== encoderSupportCheckId) {
+ return;
+ }
+
+ const supportKey = `${preflight.outputSize.width}x${preflight.outputSize.height}`;
+ outputSize = preflight.outputSize;
+ if (
+ encoderSupportKey === supportKey &&
+ (encoderSupport === 'supported' || encoderSupport === 'unsupported')
+ ) {
+ return;
+ }
+
++ if (requestedFormat !== null) {
++ format = requestedFormat;
++ encoderSupport = 'checking';
++ const canEncode = await canEncodeCapture(format, outputSize);
++ if (checkId !== encoderSupportCheckId) {
++ return;
++ }
++
++ encoderSupport = canEncode ? 'supported' : 'unsupported';
++ encoderSupportKey = supportKey;
++ setStatus(
++ canEncode
++ ? `Ready to record ${getFormatLabel(format)} at ${outputSize.width}×${outputSize.height}.`
++ : `${getFormatLabel(format)} is unavailable at ${outputSize.width}×${outputSize.height}.`,
++ !canEncode,
++ );
++ return;
++ }
++
+ encoderSupport = 'checking';
+ setStatus(
+ `Checking H.264 MP4 support at ${outputSize.width}×${outputSize.height}…`,
+ );
+ const canEncodeMp4 = await canEncodeCapture('mp4', outputSize);
+ if (checkId !== encoderSupportCheckId) {
+ return;
+ }
+
+ if (canEncodeMp4) {
+ format = 'mp4';
+ encoderSupport = 'supported';
+ encoderSupportKey = supportKey;
+ setStatus(
+ `Ready to record ${getFormatLabel(format)} at ${outputSize.width}×${outputSize.height}.`,
+ );
+ return;
+ }
+
+ setStatus(
+ `H.264 MP4 is unavailable at ${outputSize.width}×${outputSize.height}. Checking VP9 WebM…`,
+ );
+ const canEncodeWebm = await canEncodeCapture('webm', outputSize);
+ if (checkId !== encoderSupportCheckId) {
+ return;
+ }
+
+ if (canEncodeWebm) {
+ format = 'webm';
+ encoderSupport = 'supported';
+ encoderSupportKey = supportKey;
+ setStatus(
+ `H.264 MP4 is unavailable at ${outputSize.width}×${outputSize.height}. Ready to record VP9 WebM instead.`,
+ );
+ } else {
+ encoderSupport = 'unsupported';
+ encoderSupportKey = supportKey;
+ setStatus(
+ `Neither H.264 MP4 nor VP9 WebM encoding is supported at ${outputSize.width}×${outputSize.height} in this browser. Choose a lower resolution or select a smaller area.`,
+ true,
+ );
+ }
+ } catch (error) {
+ if (checkId !== encoderSupportCheckId) {
+ return;
+ }
+
+ encoderSupport = 'unsupported';
+ encoderSupportKey = null;
+ outputSize = null;
+ setStatus(error instanceof Error ? error.message : String(error), true);
+ }
+ };
+
const cancelSelection = () => {
selectionStart = null;
selecting = false;
@@@ -1505,8 -1648,19 +1677,19 @@@
return getState();
};
+ controlsOpenSetup.addEventListener('click', async () => {
+ controlsOpenSetup.disabled = true;
+ try {
+ await browser.runtime.sendMessage({type: openCaptureSetupMessageType});
+ } catch {
+ controlsErrorDescription.textContent = `Open ${canvasSetupUrl} in Chrome's address bar, then enable Canvas Draw Element and restart Chrome.`;
+ } finally {
+ controlsOpenSetup.disabled = false;
+ }
+ });
+
controlsClose.addEventListener('click', () => {
- if (capture || startingRecording || finalizing) {
+ if (capture || startingRecording || finalizing || controlsBusy) {
return;
}
@@@ -1689,331 -1843,6 +1872,338 @@@
updateControls();
});
});
+ const handleShotRequest = async (
+ name: ShotToolName,
+ input: Record<string, unknown>,
+ ) => {
++ if (!requirementsChecked) {
++ await handleRequest({
++ type: captureControllerMessageType,
++ command: 'get-state',
++ });
++ }
++
+ if (name === 'get_shot') {
+ if (input.preview !== undefined && typeof input.preview !== 'boolean') {
+ throw new Error('preview must be a boolean.');
+ }
+
+ await refreshEncoderSupport();
+ updateControls();
+ const state = getState();
+ if (!input.preview) {
+ return state;
+ }
+
+ if (
+ capture ||
+ startingRecording ||
+ finalizing ||
+ controlsBusy ||
+ selecting ||
+ selectionDrag
+ ) {
+ throw new Error(
+ 'Finish the current recording or selection before previewing the shot.',
+ );
+ }
+
+ const previewTarget = resolveCaptureTarget();
+ if (!previewTarget) {
+ throw new Error('Set up a shot before requesting a preview.');
+ }
+
+ controlsBusy = true;
+ updateControls();
+ let previewCapture: PageCapture | null = null;
+ try {
+ previewCapture = new PageCapture({
+ crop: previewTarget.crop,
+ scale,
+ format,
+ });
+ const blob = await previewCapture.preview();
+ const dataUrl = await new Promise<string>((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => resolve(String(reader.result));
+ reader.onerror = () =>
+ reject(new Error('Could not read the shot preview.'));
+ reader.readAsDataURL(blob);
+ });
+ return {
+ content: [
+ {type: 'text', text: JSON.stringify(state)},
+ {
+ type: 'image',
+ mimeType: 'image/png',
+ data: dataUrl.slice(dataUrl.indexOf(',') + 1),
+ },
+ ],
+ };
+ } finally {
+ previewCapture?.restore();
+ controlsBusy = false;
+ updateControls();
+ }
+ }
+
+ if (capture || startingRecording || finalizing || controlsBusy) {
+ throw new Error('Wait until recording and finalization have finished.');
+ }
+
+ if (name === 'save_recording') {
+ const {destination} = input;
+ if (
+ destination !== 'download' &&
+ destination !== 'new' &&
+ destination !== 'convert'
+ ) {
+ throw new Error('destination must be download, new, or convert.');
+ }
+
+ if (!completedRecording) {
+ throw new Error(
+ 'No completed recording is available. Stop recording manually before saving.',
+ );
+ }
+
+ await handleCompletedRecording(destination);
+ const state = getState();
+ if (state.error) {
+ throw new Error(state.status);
+ }
+
+ return state;
+ }
+
+ if (completedRecording) {
+ throw new Error(
+ 'Save or discard the completed recording before setting up another shot.',
+ );
+ }
+
+ const {target, resolution, format: nextFormat} = input;
+ if (
+ typeof target !== 'object' ||
+ target === null ||
+ !('type' in target)
+ ) {
+ throw new Error(
+ 'target must describe a viewport, element, or rectangle.',
+ );
+ }
+
+ let crop: CaptureCrop;
+ if (target.type === 'viewport') {
+ crop = {
+ left: 0,
+ top: 0,
+ width: window.innerWidth,
+ height: window.innerHeight,
+ };
+ } else if (target.type === 'element') {
+ if (
+ !('selector' in target) ||
+ typeof target.selector !== 'string' ||
+ !target.selector.trim()
+ ) {
+ throw new Error('An element target requires a CSS selector.');
+ }
+
+ const padding = 'padding' in target ? target.padding : 0;
+ if (
+ typeof padding !== 'number' ||
+ !Number.isFinite(padding) ||
+ padding < 0
+ ) {
+ throw new Error('padding must be a finite non-negative number.');
+ }
+
+ const elements = document.querySelectorAll(target.selector);
+ if (elements.length !== 1) {
+ throw new Error(
+ `The selector matched ${elements.length} elements. Choose a selector matching exactly one element.`,
+ );
+ }
+
+ const element = elements[0];
+ if (!element) {
+ throw new Error('The selected element is unavailable.');
+ }
+
+ const rect = element.getBoundingClientRect();
+ const elementStyle = window.getComputedStyle(element);
+ if (
+ rect.width <= 0 ||
+ rect.height <= 0 ||
+ elementStyle.visibility === 'hidden' ||
+ elementStyle.visibility === 'collapse' ||
+ Number(elementStyle.opacity) === 0 ||
+ rect.left < 0 ||
+ rect.top < 0 ||
+ rect.right > window.innerWidth ||
+ rect.bottom > window.innerHeight
+ ) {
+ throw new Error(
+ 'The element must be visible and fully inside the viewport. Scroll or prepare the page before setting up the shot.',
+ );
+ }
+
+ const left = Math.max(0, rect.left - padding);
+ const top = Math.max(0, rect.top - padding);
+ crop = {
+ left,
+ top,
+ width: Math.min(window.innerWidth, rect.right + padding) - left,
+ height: Math.min(window.innerHeight, rect.bottom + padding) - top,
+ };
+ } else if (target.type === 'rectangle') {
+ if (
+ !('left' in target) ||
+ !('top' in target) ||
+ !('width' in target) ||
+ !('height' in target) ||
+ typeof target.left !== 'number' ||
+ typeof target.top !== 'number' ||
+ typeof target.width !== 'number' ||
+ typeof target.height !== 'number'
+ ) {
+ throw new Error(
+ 'A rectangle requires numeric left, top, width, and height.',
+ );
+ }
+
+ crop = {
+ left: target.left,
+ top: target.top,
+ width: target.width,
+ height: target.height,
+ };
+ } else {
+ throw new Error('target.type must be viewport, element, or rectangle.');
+ }
+
+ if (
+ !Object.values(crop).every(Number.isFinite) ||
+ crop.left < 0 ||
+ crop.top < 0 ||
+ crop.width < minimumCaptureArea.width ||
+ crop.height < minimumCaptureArea.height ||
+ crop.left + crop.width > window.innerWidth ||
+ crop.top + crop.height > window.innerHeight
+ ) {
+ throw new Error(
+ 'The crop must be at least 10×10 CSS pixels and entirely inside the viewport.',
+ );
+ }
+
+ let outputBounds = defaultResolution;
+ if (resolution !== undefined) {
+ if (
+ typeof resolution !== 'object' ||
+ resolution === null ||
+ !('width' in resolution) ||
+ !('height' in resolution) ||
+ typeof resolution.width !== 'number' ||
+ typeof resolution.height !== 'number' ||
+ !Number.isInteger(resolution.width) ||
+ !Number.isInteger(resolution.height) ||
+ resolution.width < 2 ||
+ resolution.height < 2
+ ) {
+ throw new Error(
+ 'resolution requires integer width and height of at least 2 pixels.',
+ );
+ }
+
+ outputBounds = {width: resolution.width, height: resolution.height};
+ }
+
+ if (
+ nextFormat !== undefined &&
+ nextFormat !== 'mp4' &&
+ nextFormat !== 'webm'
+ ) {
+ throw new Error('format must be mp4 or webm.');
+ }
+
+ const nextScale = Math.min(
+ outputBounds.width / crop.width,
+ outputBounds.height / crop.height,
- maxZoom,
++ getSuggestedMaximumCaptureScale(crop.width, crop.height),
+ );
+ getCapturePreflight({crop, scale: nextScale});
+ if (selectionDrag) {
+ highlight.releasePointerCapture(selectionDrag.pointerId);
+ selectionDrag = null;
+ }
+
+ if (selecting) {
+ cancelSelection();
+ }
+
+ selectedTarget = {type: 'page-crop', crop};
+ requestedFormat = nextFormat ?? null;
+ useDefaultResolution = false;
+ scale = nextScale;
+ encoderSupportKey = null;
+ controlsDismissed = false;
+ controlsBusy = true;
+ updateHighlight();
+ updateControls();
+ try {
+ await refreshEncoderSupport();
+ return getState();
+ } finally {
+ controlsBusy = false;
+ updateControls();
+ }
+ };
+
+ // Only shot preparation, inspection, and saving cross into the page world.
+ // Recording commands stay in the extension's manual controls.
+ document.addEventListener(shotRequestEvent, (event) => {
+ if (!(event instanceof CustomEvent) || typeof event.detail !== 'string') {
+ return;
+ }
+
+ let request;
+ try {
+ request = JSON.parse(event.detail);
+ } catch {
+ return;
+ }
+
+ if (
+ typeof request?.id !== 'string' ||
+ !shotTools.some((tool) => tool.name === request.name) ||
+ typeof request.input !== 'object' ||
+ request.input === null ||
+ Array.isArray(request.input)
+ ) {
+ return;
+ }
+
+ handleShotRequest(request.name, request.input)
+ .then((result) =>
+ document.dispatchEvent(
+ new CustomEvent(shotResponseEvent, {
+ detail: JSON.stringify({id: request.id, result}),
+ }),
+ ),
+ )
+ .catch((error) =>
+ document.dispatchEvent(
+ new CustomEvent(shotResponseEvent, {
+ detail: JSON.stringify({
+ id: request.id,
+ error: error instanceof Error ? error.message : String(error),
+ }),
+ }),
+ ),
+ );
+ });
+
return {
handleRequest,
};