Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Cleanup notebooks
f-trycua5bf25c4a538d
diff --git a/libs/computer-server/computer_server/handlers/macos.py b/libs/computer-server/computer_server/handlers/macos.pyindex 86efbe3bf9..d8e2dcbb4c 100644--- a/libs/computer-server/computer_server/handlers/macos.py+++ b/libs/computer-server/computer_server/handlers/macos.py@@ -386,7 +386,7 @@ class MacOSAccessibilityHandler(BaseAccessibilityHandler):# From NSWorkspace.runningApplications docs: https://developer.apple.com/documentation/appkit/nsworkspace/runningapplications# "Similar to the NSRunningApplication class’s properties, this property will only change when the main run loop runs in a common mode"# So we need to run the main run loop to get the latest running applications- Foundation.CFRunLoopRunInMode(Foundation.kCFRunLoopDefaultMode, 0.1, False)+ Foundation.CFRunLoopRunInMode(Foundation.kCFRunLoopDefaultMode, 0.1, False) # type: ignorereturn NSWorkspace.sharedWorkspace().runningApplications()def get_ax_attribute(self, element, attribute):diff --git a/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift b/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swiftindex a412d8b1b8..83d1d99695 100644--- a/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift+++ b/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift@@ -638,6 +638,9 @@ class ImageContainerRegistry: @unchecked Sendable {try FileManager.default.moveItem(at: tempVMDir, to: URL(fileURLWithPath: vmDir.dir.path))Logger.info("Download complete: Files extracted to \(vmDir.dir.path)")+ Logger.info(+ "Run 'lume run \(vmName)' to reduce the disk image file size by using macOS sparse file system"+ )}private func copyFromCache(manifest: Manifest, manifestId: String, to destination: URL)diff --git a/notebooks/agent_nb.ipynb b/notebooks/agent_nb.ipynbindex 41cf6be53c..cc09ab7ea2 100644--- a/notebooks/agent_nb.ipynb+++ b/notebooks/agent_nb.ipynb@@ -71,7 +71,7 @@},{"cell_type": "code",- "execution_count": 10,+ "execution_count": null,"metadata": {},"outputs": [],"source": [@@ -88,13 +88,18 @@},{"cell_type": "code",- "execution_count": 1,+ "execution_count": 4,"metadata": {},"outputs": [],"source": ["import os\n",- "os.environ[\"ANTHROPIC_API_KEY\"] = \"your-anthropic-api-key\"\n",- "os.environ[\"OPENAI_API_KEY\"] = \"your-openai-api-key\""+ "\n",+ "# Get API keys from environment or prompt user\n",+ "anthropic_key = os.getenv(\"ANTHROPIC_API_KEY\") or input(\"Enter your Anthropic API key: \")\n",+ "openai_key = os.getenv(\"OPENAI_API_KEY\") or input(\"Enter your OpenAI API key: \")\n",+ "\n",+ "os.environ[\"ANTHROPIC_API_KEY\"] = anthropic_key\n",+ "os.environ[\"OPENAI_API_KEY\"] = openai_key"]},{@@ -118,8 +123,8 @@"# Create agent with Anthropic loop and provider\n","agent = ComputerAgent(\n"," computer=computer,\n",- " loop=AgentLoop.ANTHROPIC,\n",- " model=LLM(provider=LLMProvider.ANTHROPIC, name=\"claude-3-7-sonnet-20250219\"),\n",+ " loop=AgentLoop.OPENAI,\n",+ " model=LLM(provider=LLMProvider.OPENAI),\n"," save_trajectory=True,\n"," trajectory_dir=str(Path(\"trajectories\")),\n"," only_n_most_recent_images=3,\n",@@ -196,7 +201,7 @@],"metadata": {"kernelspec": {- "display_name": "Python 3",+ "display_name": "cua","language": "python","name": "python3"},@@ -210,7 +215,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.12.2"+ "version": "3.11.11"}},"nbformat": 4,diff --git a/notebooks/computer_nb.ipynb b/notebooks/computer_nb.ipynbindex c776fbfead..e37a3503ef 100644--- a/notebooks/computer_nb.ipynb+++ b/notebooks/computer_nb.ipynb@@ -6,7 +6,7 @@"source": ["## Computer\n","\n",- "This notebook demonstrates how to use Computer to operate a Lume sandbox programmatically on Apple Silicon macOS systems."+ "This notebook demonstrates how to use Computer to operate a Lume sandbox VMs programmatically on Apple Silicon macOS systems."]},{@@ -86,7 +86,7 @@"cell_type": "markdown","metadata": {},"source": [- "Pull the latest pre-built macos-sequoia-cua image. This image, based on macOS Sequoia, contains all dependencies needed to be controlled from the Computer interface."+ "Pull the latest pre-built macos-sequoia-cua image. This image, based on macOS Sequoia, contains all dependencies needed to be controlled from the cua-computer interface."]},{@@ -95,16 +95,23 @@"metadata": {},"outputs": [],"source": [- "!lume pull macos-sequoia-cua:latest"+ "!lume pull macos-sequoia-cua:latest --no-cache"]},{"cell_type": "markdown","metadata": {},"source": [- "The initial image download of thew macos-sequoia-cua image requires 80GB of storage space. However, after the first run, the image size reduces to around 20GB. Thanks to macOS's sparse file system, VM disk space is allocated dynamically - while VMs may show a total size of 50GB, they typically only consume about 20GB of physical disk space.\n",+ "Initial download requires 80GB storage, but reduces to ~30GB after first run due to macOS's sparse file system.\n","\n",- "Sandbox are stored in `~/.lume`, and locally cached images are stored in `~/.lume/cache`.\n"+ "VMs are stored in `~/.lume`, and locally cached images are stored in `~/.lume/cache`."+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "You can remove the `--no-cache` flag to also save the image to your local cache during pull (requires double the storage space). This is useful if you plan to use the same image multiple times to create other VMs."]},{@@ -215,9 +222,23 @@},{"cell_type": "code",- "execution_count": null,+ "execution_count": 7,"metadata": {},- "outputs": [],+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\n",+ "Equivalent curl command:\n",+ "curl -X POST \\\n",+ " 'http://localhost:3000/lume/vms/macos-sequoia-cua_latest/run' \\\n",+ " -H 'Content-Type: application/json' \\\n",+ " -d '{\"noDisplay\": false, \"sharedDirectories\": []}'\n",+ "\n"+ ]+ }+ ],"source": ["computer = Computer(\n"," display=\"1024x768\",\n",@@ -261,7 +282,7 @@},{"cell_type": "code",- "execution_count": 4,+ "execution_count": 8,"metadata": {},"outputs": [],"source": [@@ -374,12 +395,13 @@},{"cell_type": "code",- "execution_count": 8,+ "execution_count": null,"metadata": {},"outputs": [],"source": ["# Get accessibility tree\n",- "accessibility_tree = await computer.interface.get_accessibility_tree()"+ "accessibility_tree = await computer.interface.get_accessibility_tree()\n",+ "print(accessibility_tree)"]},{@@ -488,7 +510,7 @@],"metadata": {"kernelspec": {- "display_name": "Python 3",+ "display_name": "cua","language": "python","name": "python3"},@@ -502,7 +524,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.12.2"+ "version": "3.11.11"}},"nbformat": 4,diff --git a/notebooks/pylume_nb.ipynb b/notebooks/pylume_nb.ipynbindex 76c374199d..2e1e08f656 100644--- a/notebooks/pylume_nb.ipynb+++ b/notebooks/pylume_nb.ipynb@@ -27,6 +27,15 @@"!pip install pylume"]},+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "!pip install pydantic"+ ]+ },{"cell_type": "code","execution_count": null,@@ -41,7 +50,7 @@},{"cell_type": "code",- "execution_count": 5,+ "execution_count": 3,"metadata": {},"outputs": [],"source": [@@ -326,7 +335,7 @@],"metadata": {"kernelspec": {- "display_name": ".venv",+ "display_name": "cua","language": "python","name": "python3"},@@ -340,7 +349,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.11.7"+ "version": "3.11.11"}},"nbformat": 4,