Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
support screen recording
Abi Rajac08cf0ae571b
diff --git a/backend/poetry.lock b/backend/poetry.lockindex 61ff096e..d6b5d027 100644--- a/backend/poetry.lock+++ b/backend/poetry.lock@@ -592,13 +592,13 @@ files = [[[package]]name = "openai"-version = "1.13.3"+version = "1.14.0"description = "The official Python library for the openai API"optional = falsepython-versions = ">=3.7.1"files = [- {file = "openai-1.13.3-py3-none-any.whl", hash = "sha256:5769b62abd02f350a8dd1a3a242d8972c947860654466171d60fb0972ae0a41c"},- {file = "openai-1.13.3.tar.gz", hash = "sha256:ff6c6b3bc7327e715e4b3592a923a5a1c7519ff5dd764a83d69f633d49e77a7b"},+ {file = "openai-1.14.0-py3-none-any.whl", hash = "sha256:5c9fd3a59f5cbdb4020733ddf79a22f6b7a36d561968cb3f3dd255cdd263d9fe"},+ {file = "openai-1.14.0.tar.gz", hash = "sha256:e287057adf0ec3315abc32ddcc968d095879abd9b68bf51c0402dab13ab5ae9b"},][package.dependencies]@@ -615,13 +615,13 @@ datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"][[package]]name = "packaging"-version = "23.2"+version = "24.0"description = "Core utilities for Python packages"optional = falsepython-versions = ">=3.7"files = [- {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"},- {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"},+ {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"},+ {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"},][[package]]@@ -825,13 +825,13 @@ email = ["email-validator (>=1.0.3)"][[package]]name = "pyright"-version = "1.1.352"+version = "1.1.354"description = "Command line wrapper for pyright"optional = falsepython-versions = ">=3.7"files = [- {file = "pyright-1.1.352-py3-none-any.whl", hash = "sha256:0040cf173c6a60704e553bfd129dfe54de59cc76d0b2b80f77cfab4f50701d64"},- {file = "pyright-1.1.352.tar.gz", hash = "sha256:a621c0dfbcf1291b3610641a07380fefaa1d0e182890a1b2a7f13b446e8109a9"},+ {file = "pyright-1.1.354-py3-none-any.whl", hash = "sha256:f28d61ae8ae035fc52ded1070e8d9e786051a26a4127bbd7a4ba0399b81b37b5"},+ {file = "pyright-1.1.354.tar.gz", hash = "sha256:b1070dc774ff2e79eb0523fe87f4ba9a90550de7e4b030a2bc9e031864029a1f"},][package.dependencies]@@ -959,18 +959,18 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"][[package]]name = "setuptools"-version = "69.1.1"+version = "69.2.0"description = "Easily download, build, install, upgrade, and uninstall Python packages"optional = falsepython-versions = ">=3.8"files = [- {file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"},- {file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"},+ {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"},+ {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"},][package.extras]docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]-testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]+testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"][[package]]diff --git a/frontend/package.json b/frontend/package.jsonindex 8b4f0ee1..71094430 100644--- a/frontend/package.json+++ b/frontend/package.json@@ -45,7 +45,8 @@"tailwind-merge": "^2.0.0","tailwindcss-animate": "^1.0.7","thememirror": "^2.0.1",- "vite-plugin-checker": "^0.6.2"+ "vite-plugin-checker": "^0.6.2",+ "webm-duration-fix": "^1.0.4"},"devDependencies": {"@types/node": "^20.9.0",diff --git a/frontend/src/components/ImageUpload.tsx b/frontend/src/components/ImageUpload.tsxindex f5f5fc72..366f1049 100644--- a/frontend/src/components/ImageUpload.tsx+++ b/frontend/src/components/ImageUpload.tsx@@ -5,6 +5,8 @@ import { useDropzone } from "react-dropzone";import { toast } from "react-hot-toast";import { URLS } from "../urls";import { Badge } from "./ui/badge";+import ScreenRecorder from "./recording/ScreenRecorder";+import { ScreenRecorderState } from "../types";const baseStyle = {flex: 1,@@ -61,16 +63,23 @@ interface Props {function ImageUpload({ setReferenceImages }: Props) {const [files, setFiles] = useState<FileWithPreview[]>([]);+ // TODO: Switch to Zustand+ const [screenRecorderState, setScreenRecorderState] =+ useState<ScreenRecorderState>(ScreenRecorderState.INITIAL);+const { getRootProps, getInputProps, isFocused, isDragAccept, isDragReject } =useDropzone({maxFiles: 1,maxSize: 1024 * 1024 * 20, // 20 MBaccept: {+ // Image formats"image/png": [".png"],"image/jpeg": [".jpeg"],"image/jpg": [".jpg"],+ // Video formats"video/quicktime": [".mov"],"video/mp4": [".mp4"],+ "video/webm": [".webm"],},onDrop: (acceptedFiles) => {// Set up the preview thumbnail images@@ -154,21 +163,34 @@ function ImageUpload({ setReferenceImages }: Props) {return (<section className="container">- {/* eslint-disable-next-line @typescript-eslint/no-explicit-any */}- <div {...getRootProps({ style: style as any })}>- <input {...getInputProps()} />- <p className="text-slate-700 text-lg">- Drag & drop a screenshot here, <br />- or click to upload- </p>- </div>- <div className="text-center text-sm text-slate-800 mt-4">- <Badge>New!</Badge> Upload a screen recording in .mp4 or .mov format to- clone a whole app (experimental).{" "}- <a className="underline" href={URLS["intro-to-video"]} target="_blank">- Learn more.- </a>- </div>+ {screenRecorderState === ScreenRecorderState.INITIAL && (+ /* eslint-disable-next-line @typescript-eslint/no-explicit-any */+ <div {...getRootProps({ style: style as any })}>+ <input {...getInputProps()} />+ <p className="text-slate-700 text-lg">+ Drag & drop a screenshot here, <br />+ or click to upload+ </p>+ </div>+ )}+ {screenRecorderState === ScreenRecorderState.INITIAL && (+ <div className="text-center text-sm text-slate-800 mt-4">+ <Badge>New!</Badge> Upload a screen recording (.mp4, .mov) or record+ your screen to clone a whole app (experimental).{" "}+ <a+ className="underline"+ href={URLS["intro-to-video"]}+ target="_blank"+ >+ Learn more.+ </a>+ </div>+ )}+ <ScreenRecorder+ screenRecorderState={screenRecorderState}+ setScreenRecorderState={setScreenRecorderState}+ generateCode={setReferenceImages}+ /></section>);}diff --git a/frontend/src/components/recording/ScreenRecorder.tsx b/frontend/src/components/recording/ScreenRecorder.tsxnew file mode 100644index 00000000..c2cc06dc--- /dev/null+++ b/frontend/src/components/recording/ScreenRecorder.tsx@@ -0,0 +1,115 @@+import { useState } from "react";+import { Button } from "../ui/button";+import { ScreenRecorderState } from "../../types";+import { blobToBase64DataUrl } from "./utils";+import fixWebmDuration from "webm-duration-fix";+import toast from "react-hot-toast";++interface Props {+ screenRecorderState: ScreenRecorderState;+ setScreenRecorderState: (state: ScreenRecorderState) => void;+ generateCode: (+ referenceImages: string[],+ inputMode: "image" | "video"+ ) => void;+}++function ScreenRecorder({+ screenRecorderState,+ setScreenRecorderState,+ generateCode,+}: Props) {+ const [mediaRecorder, setMediaRecorder] = useState<MediaRecorder | null>(+ null+ );+ const [screenRecordingDataUrl, setScreenRecordingDataUrl] = useState<+ string | null+ >(null);++ const startScreenRecording = async () => {+ try {+ // Get the screen recording stream+ const stream = await navigator.mediaDevices.getDisplayMedia({+ video: true,+ audio: { echoCancellation: true },+ });++ // TODO: Test across different browsers+ // Create the media recorder+ const options = { mimeType: "video/webm" };+ const mediaRecorder = new MediaRecorder(stream, options);+ setMediaRecorder(mediaRecorder);++ const chunks: BlobPart[] = [];++ // Accumalate chunks as data is available+ mediaRecorder.ondataavailable = (e: BlobEvent) => chunks.push(e.data);++ // When media recorder is stopped, create a data URL+ mediaRecorder.onstop = async () => {+ // TODO: Do I need to fix duration if it's not a webm?+ const completeBlob = await fixWebmDuration(+ new Blob(chunks, {+ type: options.mimeType,+ })+ );++ const dataUrl = await blobToBase64DataUrl(completeBlob);+ setScreenRecordingDataUrl(dataUrl);+ setScreenRecorderState(ScreenRecorderState.FINISHED);+ };++ // Start recording+ mediaRecorder.start();+ setScreenRecorderState(ScreenRecorderState.RECORDING);+ } catch (error) {+ toast.error("Could not start screen recording");+ throw error;+ }+ };++ const stopScreenRecording = () => {+ if (mediaRecorder) {+ mediaRecorder.stop();+ setMediaRecorder(null);+ }+ };++ const kickoffGeneration = () => {+ if (screenRecordingDataUrl) {+ generateCode([screenRecordingDataUrl], "video");+ } else {+ toast.error("Screen recording does not exist. Please try again.");+ throw new Error("No screen recording data url");+ }+ };++ return (+ <div className="flex items-center justify-center my-3">+ {screenRecorderState === ScreenRecorderState.INITIAL && (+ <Button onClick={startScreenRecording}>Record Screen</Button>+ )}++ {screenRecorderState === ScreenRecorderState.RECORDING && (+ <div className="flex items-center flex-col gap-y-4">+ <div className="flex items-center mr-2 text-xl gap-x-1">+ <span className="block h-10 w-10 bg-red-600 rounded-full mr-1 animate-pulse"></span>+ <span>Recording...</span>+ </div>+ <Button onClick={stopScreenRecording}>Finish Recording</Button>+ </div>+ )}++ {screenRecorderState === ScreenRecorderState.FINISHED && (+ <div className="flex items-center flex-col gap-y-4">+ <div className="flex items-center mr-2 text-xl gap-x-1">+ <span>Screen Recording Captured.</span>+ </div>+ <Button onClick={kickoffGeneration}>Generate</Button>+ </div>+ )}+ </div>+ );+}++export default ScreenRecorder;diff --git a/frontend/src/components/recording/utils.ts b/frontend/src/components/recording/utils.tsnew file mode 100644index 00000000..b482ecfc--- /dev/null+++ b/frontend/src/components/recording/utils.ts@@ -0,0 +1,31 @@+export function downloadBlob(blob: Blob) {+ // Create a URL for the blob object+ const videoURL = URL.createObjectURL(blob);++ // Create a temporary anchor element and trigger the download+ const a = document.createElement("a");+ a.href = videoURL;+ a.download = "recording.webm";+ document.body.appendChild(a);+ a.click();+ document.body.removeChild(a);++ // Clear object URL+ URL.revokeObjectURL(videoURL);+}++export function blobToBase64DataUrl(blob: Blob): Promise<string> {+ return new Promise((resolve, reject) => {+ const reader = new FileReader();+ reader.onloadend = () => {+ if (reader.result) {+ resolve(reader.result as string);+ } else {+ reject(new Error("FileReader did not return a result."));+ }+ };+ reader.onerror = () =>+ reject(new Error("FileReader encountered an error."));+ reader.readAsDataURL(blob);+ });+}diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex 89440fee..fd460822 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -25,6 +25,12 @@ export enum AppState {CODE_READY = "CODE_READY",}+export enum ScreenRecorderState {+ INITIAL = "initial",+ RECORDING = "recording",+ FINISHED = "finished",+}+export interface CodeGenerationParams {generationType: "create" | "update";inputMode: "image" | "video";diff --git a/frontend/yarn.lock b/frontend/yarn.lockindex 5f098859..1edf79e7 100644--- a/frontend/yarn.lock+++ b/frontend/yarn.lock@@ -1610,6 +1610,11 @@ base64-arraybuffer@^1.0.2:resolved "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz#1c37589a7c4b0746e34bd1feb951da2df01c1bdc"integrity sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==+base64-js@^1.3.1:+ version "1.5.1"+ resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"+ integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==+binary-extensions@^2.0.0:version "2.2.0"resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz"@@ -1657,6 +1662,14 @@ buffer-from@^1.0.0:resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==+buffer@^6.0.3:+ version "6.0.3"+ resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6"+ integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==+ dependencies:+ base64-js "^1.3.1"+ ieee754 "^1.2.1"+cac@^6.7.14:version "6.7.14"resolved "https://registry.yarnpkg.com/cac/-/cac-6.7.14.tgz#804e1e6f506ee363cb0e3ccbb09cad5dd9870959"@@ -1993,6 +2006,11 @@ dotenv@^16.0.0:resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.3.1.tgz#369034de7d7e5b120972693352a3bf112172cc3e"integrity sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==+ebml-block@^1.1.2:+ version "1.1.2"+ resolved "https://registry.yarnpkg.com/ebml-block/-/ebml-block-1.1.2.tgz#fd49951b0faf5a3049bdd61c851a76b5e679c290"+ integrity sha512-HgNlIsRFP6D9VKU5atCeHRJY7XkJP8bOe8yEhd8NB7B3b4++VWTyauz6g650iiPmLfPLGlVpoJmGSgMfXDYusg==+ejs@^3.1.6:version "3.1.9"resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.9.tgz#03c9e8777fe12686a9effcef22303ca3d8eeb361"@@ -2186,6 +2204,11 @@ esutils@^2.0.2:resolved "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz"integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==+events@^3.3.0:+ version "3.3.0"+ resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400"+ integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==+execa@^8.0.1:version "8.0.1"resolved "https://registry.yarnpkg.com/execa/-/execa-8.0.1.tgz#51f6a5943b580f963c3ca9c6321796db8cc39b8c"@@ -2467,6 +2490,11 @@ human-signals@^5.0.0:resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-5.0.0.tgz#42665a284f9ae0dade3ba41ebc37eb4b852f3a28"integrity sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==+ieee754@^1.2.1:+ version "1.2.1"+ resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"+ integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==+ignore@^5.2.0, ignore@^5.2.4:version "5.2.4"resolved "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz"@@ -2498,6 +2526,11 @@ inherits@2:resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz"integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==+int64-buffer@^1.0.1:+ version "1.0.1"+ resolved "https://registry.yarnpkg.com/int64-buffer/-/int64-buffer-1.0.1.tgz#c78d841b444cadf036cd04f8683696c740f15dca"+ integrity sha512-+3azY4pXrjAupJHU1V9uGERWlhoqNswJNji6aD/02xac7oxol508AsMC5lxKhEqyZeDFy3enq5OGWXF4u75hiw==+invariant@^2.2.4:version "2.2.4"resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6"@@ -3765,6 +3798,16 @@ w3c-keyname@^2.2.4:resolved "https://registry.yarnpkg.com/w3c-keyname/-/w3c-keyname-2.2.8.tgz#7b17c8c6883d4e8b86ac8aba79d39e880f8869c5"integrity sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==+webm-duration-fix@^1.0.4:+ version "1.0.4"+ resolved "https://registry.yarnpkg.com/webm-duration-fix/-/webm-duration-fix-1.0.4.tgz#fef235cb3d3ed3363507f705a7577dbb9fdedae6"+ integrity sha512-kvhmSmEnuohtK+j+mJswqCCM2ViKb9W8Ch0oAxcaeUvpok5CsMORQLnea+CYKDXPG6JH12H0CbRK85qhfeZLew==+ dependencies:+ buffer "^6.0.3"+ ebml-block "^1.1.2"+ events "^3.3.0"+ int64-buffer "^1.0.1"+which@^2.0.1:version "2.0.2"resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz"