Skip to content

Desktop app that records your on-screen work session and uses the GitHub Copilot SDK to reconstruct it as an intent + ordered steps, then builds a reusable Skill or Automation for Microsoft Autopilots like Scout, Microsoft Copilot Cowork, or Copilot Studio.

Implement Windows URL provider via UI Automation sidecar

Adi Leibowitz (from Dev Box)e1b9c88b3bbb
diff --git a/electron/collectors/url-provider.ts b/electron/collectors/url-provider.ts
index 0e3f9b6..43166e6 100644
--- a/electron/collectors/url-provider.ts
+++ b/electron/collectors/url-provider.ts
@@ -1,4 +1,5 @@
-import { execFile } from "node:child_process";
+import { execFile, spawn, type ChildProcess } from "node:child_process";
+import { createInterface, type Interface as ReadlineInterface } from "node:readline";
/**
* Reads the *active tab URL* of the frontmost browser — the single richest
@@ -116,9 +117,193 @@ export class MacUrlProvider implements UrlProvider {
}
}
+/**
+ * Windows URL provider via a persistent PowerShell sidecar running UI Automation.
+ *
+ * Architecture: a single long-lived PowerShell process loads the UIA assemblies
+ * once at startup, then enters a loop reading commands from stdin. Each `get()`
+ * call writes a request line; the sidecar reads the active browser's address bar
+ * via AutomationId (omnibox, addressEditBox, urlbar) with fallbacks to Edit and
+ * Document controls, and writes the result as a single JSON line to stdout.
+ *
+ * This mirrors the approach used by other open-source Windows activity trackers
+ * (f4w4z/trace, Sunhaiy/suxin-desktop) which also use a persistent PowerShell
+ * process to avoid the ~200ms cold-start penalty per query.
+ */
+export class WindowsUrlProvider implements UrlProvider {
+ private proc: ChildProcess | null = null;
+ private reader: ReadlineInterface | null = null;
+ private pending: { resolve: (v: string) => void; timer: NodeJS.Timeout } | null = null;
+ private starting = false;
+
+ supports(app: string): boolean {
+ return isBrowserApp(app);
+ }
+
+ async get(_app: string): Promise<ActiveUrl | null> {
+ this.ensureProc();
+ if (!this.proc) return null;
+ try {
+ const line = await this.query();
+ if (!line) return null;
+ const parsed = JSON.parse(line) as { url?: string; title?: string };
+ if (!parsed.url) return null;
+ const cleanTitle = parsed.title
+ ?.replace(/\s*[-\u2013\u2014]\s*(Microsoft\s+Edge|Google\s+Chrome|Brave|Vivaldi|Opera|Arc).*$/i, "")
+ .trim();
+ return { url: parsed.url, title: cleanTitle || undefined };
+ } catch {
+ return null;
+ }
+ }
+
+ private ensureProc(): void {
+ if (this.proc || this.starting) return;
+ this.starting = true;
+ this.proc = spawn("powershell", ["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", SIDECAR_SCRIPT], {
+ stdio: ["pipe", "pipe", "ignore"],
+ windowsHide: true,
+ });
+ this.proc.on("exit", () => {
+ this.proc = null;
+ this.reader = null;
+ this.starting = false;
+ });
+ this.proc.on("error", () => {
+ this.proc = null;
+ this.reader = null;
+ this.starting = false;
+ });
+ this.reader = createInterface({ input: this.proc.stdout!, crlfDelay: Infinity });
+ this.reader.on("line", (line: string) => {
+ if (!this.pending) return;
+ clearTimeout(this.pending.timer);
+ this.pending.resolve(line);
+ this.pending = null;
+ });
+ this.starting = false;
+ }
+
+ private query(): Promise<string> {
+ return new Promise((resolve) => {
+ if (!this.proc?.stdin?.writable) {
+ resolve("");
+ return;
+ }
+ const timer = setTimeout(() => {
+ this.pending = null;
+ resolve("");
+ }, 1500);
+ this.pending = { resolve, timer };
+ this.proc.stdin.write("query\n");
+ });
+ }
+}
+
+/**
+ * Self-contained PowerShell script that:
+ * 1. Loads UIA assemblies and Win32 interop once at startup
+ * 2. Reads lines from stdin in a loop
+ * 3. On each "query" line, finds the foreground browser URL via UIA
+ * 4. Writes one JSON line to stdout: {"url":"...","title":"..."} or {}
+ *
+ * The script uses AutomationId-based lookup (omnibox, addressEditBox, urlbar)
+ * which is more reliable than name-matching across locales.
+ */
+const SIDECAR_SCRIPT = `
+$OutputEncoding = [System.Text.Encoding]::UTF8
+[Console]::OutputEncoding = [System.Text.Encoding]::UTF8
+Add-Type -AssemblyName UIAutomationClient
+Add-Type -AssemblyName UIAutomationTypes
+
+function Get-BrowserUrl {
+ # The caller (ActiveWindowCollector) already knows a browser is frontmost.
+ # Find the first Chrome_WidgetWin_1 top-level window and read its address bar.
+ $root = [System.Windows.Automation.AutomationElement]::RootElement
+ $cond = New-Object System.Windows.Automation.PropertyCondition(
+ [System.Windows.Automation.AutomationElement]::ClassNameProperty, 'Chrome_WidgetWin_1')
+ $wins = $root.FindAll([System.Windows.Automation.TreeScope]::Children, $cond)
+ $el = $null
+ foreach ($w in $wins) { $el = $w; break }
+ if (-not $el) { return $null }
+
+ try {
+
+ # Primary: known automation IDs for address bars
+ foreach ($id in @('omnibox','addressEditBox','urlbar-edit-view','urlbar','view_id/omnibox_url_field')) {
+ $cond = New-Object System.Windows.Automation.PropertyCondition(
+ [System.Windows.Automation.AutomationElement]::AutomationIdProperty, $id)
+ $found = $el.FindFirst([System.Windows.Automation.TreeScope]::Descendants, $cond)
+ if ($found) {
+ try {
+ $vp = $found.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern)
+ if ($vp) {
+ $val = ([System.Windows.Automation.ValuePattern]$vp).Current.Value
+ if ($val -match '^https?://') {
+ return @{ url = $val; title = $el.Current.Name }
+ }
+ }
+ } catch {}
+ }
+ }
+
+ # Fallback: Edit control whose name matches address/URL patterns
+ $editCond = New-Object System.Windows.Automation.PropertyCondition(
+ [System.Windows.Automation.AutomationElement]::ControlTypeProperty,
+ [System.Windows.Automation.ControlType]::Edit)
+ $edits = $el.FindAll([System.Windows.Automation.TreeScope]::Descendants, $editCond)
+ foreach ($edit in $edits) {
+ if ($edit.Current.Name -match 'address|search bar|URL') {
+ try {
+ $vp = $edit.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern)
+ if ($vp) {
+ $val = ([System.Windows.Automation.ValuePattern]$vp).Current.Value
+ if ($val -match '^https?://') {
+ return @{ url = $val; title = $el.Current.Name }
+ }
+ }
+ } catch {}
+ }
+ }
+
+ # Fallback 2: Document control (newer Edge/Chrome versions)
+ $docCond = New-Object System.Windows.Automation.PropertyCondition(
+ [System.Windows.Automation.AutomationElement]::ControlTypeProperty,
+ [System.Windows.Automation.ControlType]::Document)
+ $docs = $el.FindAll([System.Windows.Automation.TreeScope]::Descendants, $docCond)
+ foreach ($d in $docs) {
+ try {
+ $vp = $d.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern)
+ if ($vp) {
+ $val = ([System.Windows.Automation.ValuePattern]$vp).Current.Value
+ if ($val -match '^https?://') {
+ return @{ url = $val; title = $el.Current.Name }
+ }
+ }
+ } catch {}
+ }
+ } catch {}
+ return $null
+}
+
+# Main loop: read stdin, respond on stdout
+while ($true) {
+ $line = [Console]::In.ReadLine()
+ if ($null -eq $line) { break }
+ $result = Get-BrowserUrl
+ if ($result) {
+ $json = '{"url":' + (ConvertTo-Json $result.url -Compress) + ',"title":' + (ConvertTo-Json $result.title -Compress) + '}'
+ [Console]::Out.WriteLine($json)
+ } else {
+ [Console]::Out.WriteLine('{}')
+ }
+ [Console]::Out.Flush()
+}
+`;
+
/** The URL provider for the current platform, or null where unsupported. */
export function createUrlProvider(): UrlProvider | null {
if (process.platform === "darwin") return new MacUrlProvider();
- // Windows: UI Automation address-bar read is a later milestone.
+ if (process.platform === "win32") return new WindowsUrlProvider();
return null;
}