Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
//! CUA JUKEBOX orchestrator — coordinated multi-cursor "computer-use" music.
//!
//! Reads a MIDI file (or a built-in demo song), turns each track into its own
//! instrument window (a miniwob-style minigame), and gives each one its OWN
//! cua-driver session = its OWN uniquely-coloured agent cursor. While the song
//! plays, every note steers that track's cursor onto its widget and clicks it
//! in the background — the click is what makes the sound. One cursor per part,
//! one colour per agent, all driven off a single clock: the dumbest possible
//! orchestra, performed entirely by background computer-use.
//!
//! Timing is locked with the cursor's `glide_duration_ms` (set per session via
//! `set_agent_cursor_motion`): every glide takes a known, fixed time regardless
//! of how far the cursor must travel across its pitch strip, so the orchestrator
//! can pre-roll each click by exactly that lead and land the actuation on the
//! beat. (That field is honoured identically on macOS and Windows — it lives in
//! the shared render core.)
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::thread;
use std::time::{Duration, Instant};
use windows::Win32::Foundation::{BOOL, HANDLE, HWND, LPARAM, POINT, RECT, TRUE};
use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS};
use windows::Win32::Graphics::Gdi::ClientToScreen;
use windows::Win32::System::JobObjects::{
AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation,
SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
};
use windows::Win32::UI::WindowsAndMessaging::{
EnumWindows, GetClientRect, GetWindowTextW, GetWindowThreadProcessId, IsWindowVisible,
SetForegroundWindow, SetWindowPos, HWND_TOP, SWP_NOACTIVATE, SWP_NOZORDER, SWP_SHOWWINDOW,
};
// Widget geometry — MUST match jukebox-app's WX0/WX1/WY0/WY1/KEYS_SPAN.
const WX0: f64 = 0.06;
const WX1: f64 = 0.94;
const WY0: f64 = 0.34;
const WY1: f64 = 0.92;
const KEYS_SPAN: i32 = 24;
const MAX_TRACKS: usize = 9;
// One cua-driver palette per agent. Keying a session to a palette NAME makes
// that session's overlay cursor render in that palette automatically
// (`Palette::for_instance(session)`), exactly like the sibling multi-cursor
// demo. The hex is that palette's mid colour, reused for the instrument window
// accent AND the transport legend, so the cursor, its window, and the legend
// all read as the same colour. (`cursor_color` on `set_agent_cursor_motion`
// only updates registry state, not the Windows overlay paint — so we rely on
// the session-name palette instead of fighting it.)
const SESSIONS: [(&str, &str); 9] = [
("crimson", "#e85262"),
("amber", "#f4b242"),
("aqua", "#4ccce0"),
("mint_lime", "#60daae"),
("orchid", "#dd71ec"),
("soft_purple", "#b284ff"),
("rose_gold", "#f784aa"),
("chartreuse", "#b8dc36"),
("cobalt", "#507eec"),
];
#[derive(Clone, Copy, PartialEq)]
enum Kind {
Pad,
Keys,
Drums,
}
#[derive(Clone)]
struct Note {
t: f64,
pitch: u8,
/// Carried from the source for fidelity; the instrument picks its own hit
/// velocity (a background click can't carry one), so it's unused here.
#[allow(dead_code)]
vel: u8,
}
struct Track {
name: String,
notes: Vec<Note>,
kind: Kind,
wave: &'static str,
root: i32,
/// Number of semitones the pitch strip spans (fit to the track's range so
/// wide melodies aren't clamped to the top key). Drums ignore it.
span: i32,
color: &'static str,
session: String,
title: String,
hwnd: HWND,
pid: u32,
/// True when this track is percussion (most notes on MIDI channel 10, or a
/// drum-kit name) → rendered as a 3-zone kick/snare/hat pad.
is_drum: bool,
}
struct Song {
tracks: Vec<Track>,
bpm: u32,
dur_sec: f64,
}
/// Send-able snapshot of one cursor's worth of work (a "voice" — a single pool
/// member of a track). HWND is carried as an isize because the raw handle
/// pointer isn't `Send`. Several voices can share one track's window/colour.
struct Voice {
pid: u32,
hwnd_addr: isize,
session: String,
kind: Kind,
root: i32,
span: i32,
notes: Arc<Vec<Note>>,
}
/// Max same-colour cursors a single track may grow to (a chord wider than this
/// reuses cursor 0 and accepts a little overlap rather than spawning forever).
const MAX_POOL: usize = 6;
/// Assign each (time-sorted) note to a pool cursor: try the lowest-indexed
/// cursor that's free (its previous glide finished ≥ now), else grow the pool,
/// else (at the cap) reuse cursor 0. `busy` is the glide window a cursor is
/// "occupied" for after it fires. Returns the per-note cursor index and the
/// resulting pool size. This is the deterministic form of "try the first, go to
/// the next if occupied during the 100ms, spawn one if none free".
fn assign_pool(notes: &[Note], busy: f64, cap: usize) -> (Vec<usize>, usize) {
let mut free_at: Vec<f64> = Vec::new();
let mut assign = Vec::with_capacity(notes.len());
for n in notes {
let chosen = free_at.iter().position(|&f| f <= n.t + 1e-6);
let i = match chosen {
Some(i) => i,
None if free_at.len() < cap => {
free_at.push(0.0);
free_at.len() - 1
}
None => 0,
};
free_at[i] = n.t + busy;
assign.push(i);
}
(assign, free_at.len().max(1))
}
/// Blend a `#rrggbb` toward white by `t` (0=unchanged, 1=white) for a gradient
/// tip stop, so a forced-colour cursor still reads as a lit arrow, not a flat blob.
fn lighten(hex: &str, t: f64) -> String {
let v = u32::from_str_radix(hex.trim_start_matches('#'), 16).unwrap_or(0xffffff);
let f = |sh: u32| {
let c = ((v >> sh) & 0xff) as f64;
(c + (255.0 - c) * t) as u32
};
format!("#{:02x}{:02x}{:02x}", f(16), f(8), f(0))
}
// ── kill-on-exit job (whole tree dies with the orchestrator) ───────────────────
static JOB: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
fn job() -> HANDLE {
let raw = *JOB.get_or_init(|| unsafe {
let h = CreateJobObjectW(None, windows::core::PCWSTR::null()).expect("CreateJobObjectW");
let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
let _ = SetInformationJobObject(
h,
JobObjectExtendedLimitInformation,
&info as *const _ as *const core::ffi::c_void,
std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32,
);
h.0 as usize
});
HANDLE(raw as *mut core::ffi::c_void)
}
fn assign_to_job(child: &Child) {
use std::os::windows::io::AsRawHandle;
unsafe {
let _ = AssignProcessToJobObject(
job(),
HANDLE(child.as_raw_handle() as *mut core::ffi::c_void),
);
}
}
fn main() {
let demo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.to_path_buf();
let repo_root = demo_root.parent().unwrap().parent().unwrap().to_path_buf();
let cua = std::env::var("CUA_DRIVER_EXE")
.map(PathBuf::from)
.unwrap_or_else(|_| repo_root.join("libs/cua-driver/rust/target/debug/cua-driver.exe"));
let app = std::env::var("JUKEBOX_APP_EXE")
.map(PathBuf::from)
.unwrap_or_else(|_| demo_root.join("target/debug/jukebox-app.exe"));
if !cua.exists() {
eprintln!("cua-driver.exe not found at {cua:?}");
std::process::exit(1);
}
if !app.exists() {
eprintln!("jukebox-app.exe not found at {app:?} — run `cargo build` first");
std::process::exit(1);
}
// Raise the system timer resolution to 1ms so the per-note `thread::sleep`
// that schedules each click is accurate to ~1ms instead of Windows' default
// ~15ms — the dominant residual timing jitter once the per-note process
// spawn was removed. Process-global; the OS restores it on exit.
unsafe {
let _ = windows::Win32::Media::timeBeginPeriod(1);
}
let args: Vec<String> = std::env::args().collect();
let auto = args.iter().any(|a| a == "--auto");
let midi_path = args
.iter()
.skip(1)
.find(|a| a.to_lowercase().ends_with(".mid") || a.to_lowercase().ends_with(".midi"));
let glide_ms: u64 = std::env::var("JUKEBOX_GLIDE_MS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(200);
// The dominant actuation latency IS the glide, so default the pre-roll to it
// (the per-voice EMA refines from there); overridable with JUKEBOX_LEAD_MS.
let lead = Duration::from_millis(
std::env::var("JUKEBOX_LEAD_MS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(glide_ms),
);
let mut song = match midi_path {
Some(p) => match load_midi(p) {
Ok(s) => s,
Err(e) => {
eprintln!("[orch] MIDI parse failed ({e}); using demo song");
demo_song()
}
},
None => demo_song(),
};
assign_roles(&mut song);
if song.tracks.is_empty() {
eprintln!("[orch] no tracks with notes");
std::process::exit(1);
}
eprintln!(
"[orch] {} parts @ {} bpm, {:.1}s:",
song.tracks.len(),
song.bpm,
song.dur_sec
);
for t in &song.tracks {
let kind = match t.kind {
Kind::Drums => "drums",
Kind::Pad => "pad",
Kind::Keys => "keys",
};
let lo = t.notes.iter().map(|n| n.pitch as i32).min().unwrap_or(0);
let hi = t.notes.iter().map(|n| n.pitch as i32).max().unwrap_or(0);
let range = hi - lo + 1;
let fit = if t.kind == Kind::Keys && range > t.span {
format!(
" RANGE {range}st > strip {} → {} note(s) clamped",
t.span,
t.notes
.iter()
.filter(|n| (n.pitch as i32 - t.root) >= t.span)
.count()
)
} else {
String::new()
};
eprintln!(
" · {:<22} {:<6} {:<8} {:>4} notes range {}-{} ({}st), strip {}st{}",
t.name,
kind,
t.wave,
t.notes.len(),
lo,
hi,
range,
t.span,
fit
);
}
// Reap any leftover daemon from a previous run FIRST. Its job object only
// fires KILL_ON_JOB_CLOSE when the *owning* orchestrator exits cleanly, so a
// hard-killed run can leave a `cua-driver serve` alive — and a stale daemon
// keeps its own full-screen overlay, whose agent0..N cursors stack on top of
// this run's at the same (deterministic) coordinates. That's what makes it
// look like more than one cursor per window. Start from a clean slate.
for img in ["cua-driver.exe", "jukebox-app.exe"] {
let _ = Command::new("taskkill")
.args(["/F", "/IM", img])
.stdout(Stdio::null())
.stderr(Stdio::null())
.status();
}
// Give the killed daemon time to release its named pipe before we start a
// fresh one — too short a wait races the pipe teleport and can wedge the new
// daemon's first connections (observed as a one-off ~10s-late performance).
thread::sleep(Duration::from_millis(800));
// Start the cua-driver daemon.
eprintln!("[orch] starting cua-driver daemon…");
let mut daemon = Command::new(&cua)
.arg("serve")
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("spawn cua-driver serve");
assign_to_job(&daemon);
thread::sleep(Duration::from_millis(1500));
// Launch the controller (foreground) + one instrument window per track.
let mut controller = Command::new(&app)
.args([
"controller",
"--title",
"CUA JUKEBOX — Transport",
"--color",
"#3bd0ff",
"--bpm",
&song.bpm.to_string(),
"--dur-ms",
&((song.dur_sec * 1000.0) as u64).to_string(),
"--tracks",
&song
.tracks
.iter()
.map(|t| format!("{}|{}", t.name, t.color))
.collect::<Vec<_>>()
.join(","),
])
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.expect("spawn controller");
assign_to_job(&controller);
let controller_out = controller.stdout.take().unwrap();
let mut kids: Vec<Child> = Vec::new();
for t in &song.tracks {
let mut c = Command::new(&app);
c.args([
"instrument",
"--title",
&t.title,
"--kind",
match t.kind {
Kind::Keys => "keys",
Kind::Drums => "drums",
Kind::Pad => "pad",
},
"--wave",
t.wave,
"--color",
t.color,
"--label",
&t.name,
"--root",
&t.root.to_string(),
"--span",
&t.span.to_string(),
]);
if let Ok(ch) = c.stdout(Stdio::null()).stderr(Stdio::null()).spawn() {
assign_to_job(&ch);
kids.push(ch);
}
}
thread::sleep(Duration::from_millis(1400)); // window warmup
// Discover HWNDs/pids.
for t in song.tracks.iter_mut() {
if let Some((h, pid)) = find_window_by_title(&t.title) {
t.hwnd = h;
t.pid = pid;
} else {
eprintln!("[orch] (warn) no window for '{}'", t.title);
}
}
song.tracks.retain(|t| !t.hwnd.0.is_null());
let controller_hwnd = find_window_by_title("CUA JUKEBOX — Transport").map(|(h, _)| h);
// Layout: a 600×80 transport bar above a tight grid of fixed 200×160 tiles,
// packed GAP px apart and centered in the work area.
const TILE_W: i32 = 200;
const TILE_H: i32 = 160;
const CTRL_W: i32 = 600;
const CTRL_H: i32 = 80;
const GAP: i32 = 6;
let wa = work_area();
let n = song.tracks.len() as i32;
let cols = (n as f64).sqrt().ceil().max(1.0) as i32;
let rows = (n + cols - 1) / cols.max(1);
let grid_w = cols * TILE_W + (cols - 1) * GAP;
let grid_h = rows * TILE_H + (rows - 1) * GAP;
let block_w = grid_w.max(CTRL_W);
let block_h = CTRL_H + GAP + grid_h;
let ox = wa.left + (((wa.right - wa.left) - block_w) / 2).max(0);
let oy = wa.top + (((wa.bottom - wa.top) - block_h) / 2).max(0);
if let Some(ch) = controller_hwnd {
place(ch, ox + (block_w - CTRL_W) / 2, oy, CTRL_W, CTRL_H, true);
}
let gy = oy + CTRL_H + GAP;
let gx = ox + (block_w - grid_w) / 2;
for (i, t) in song.tracks.iter().enumerate() {
let (col, row) = (i as i32 % cols, i as i32 / cols);
place(
t.hwnd,
gx + col * (TILE_W + GAP),
gy + row * (TILE_H + GAP),
TILE_W,
TILE_H,
false,
);
}
if let Some(ch) = controller_hwnd {
unsafe {
let _ = SetForegroundWindow(ch);
}
}
thread::sleep(Duration::from_millis(700));
// Build per-track cursor POOLS and arm each member. A track usually needs
// one cursor, but when notes land within the glide window (e.g. a chord) the
// first cursor is still mid-glide, so the next free pool cursor takes the
// note — growing the pool. Every member of a track is forced to the SAME
// colour (via set_agent_cursor_style's gradient — cursor_color on
// set_agent_cursor_motion is registry-only on the Windows overlay and would
// not repaint), so a chord fans out into several identically-coloured
// cursors on that one window.
// A cursor is "occupied" for the whole click — the glide PLUS the dispatch
// overhead (IPC + click-post + arrival frame), not just the glide. Sizing
// the pool against the real click duration means any track whose notes are
// closer together than that spawns enough cursors to keep up, instead of one
// cursor falling progressively behind.
let busy_sec = (glide_ms as f64 + 130.0) / 1000.0;
let mut voices: Vec<Voice> = Vec::new();
for t in &song.tracks {
let (assign, pool) = assign_pool(&t.notes, busy_sec, MAX_POOL);
let tip = lighten(t.color, 0.5);
for k in 0..pool {
let session = format!("{}_{}", t.session, k);
let _ = run_call(
&cua,
"set_agent_cursor_enabled",
&format!(r#"{{"enabled":true,"session":"{session}"}}"#),
);
// Fixed 100ms glide so every actuation lands on the beat regardless
// of travel distance (and so "occupied for the glide window" has a
// known length the pool sizes against).
// turn_radius 40 = half the default 80 → tighter glide curves, which
// read better in the small tiles.
let _ = run_call(
&cua,
"set_agent_cursor_motion",
&format!(
r#"{{"session":"{session}","cursor_label":"{}","cursor_size":15,"glide_duration_ms":{glide_ms},"spring":1.0,"arc_size":0.08,"dwell_after_click_ms":0,"idle_hide_ms":0,"turn_radius":40}}"#,
t.name
),
);
let _ = run_call(
&cua,
"set_agent_cursor_style",
&format!(
r#"{{"session":"{session}","gradient_colors":["{tip}","{}"],"bloom_color":"{}"}}"#,
t.color, t.color
),
);
let notes_k: Vec<Note> = t
.notes
.iter()
.zip(assign.iter())
.filter(|(_, &a)| a == k)
.map(|(n, _)| n.clone())
.collect();
voices.push(Voice {
pid: t.pid,
hwnd_addr: t.hwnd.0 as isize,
session,
kind: t.kind,
root: t.root,
span: t.span,
notes: Arc::new(notes_k),
});
}
if pool > 1 {
eprintln!(
"[orch] {:<10} pool={} same-colour cursors (notes closer than {:.0}ms)",
t.name,
pool,
busy_sec * 1000.0
);
}
}
let plans: Arc<Vec<Voice>> = Arc::new(voices);
let cua = Arc::new(cua);
let playing = Arc::new(AtomicBool::new(false));
let generation = Arc::new(AtomicU64::new(0));
let dur_sec = song.dur_sec;
let base_lead_ms = lead.as_secs_f64() * 1000.0;
// `timing` collects the signed per-note error (actual−scheduled) for the
// end-of-song report; cleared each performance. The adaptive lead itself is
// PER-VOICE (a thread-local EMA below), so one congested track — e.g. the
// Pad pool firing a triad — can't skew the lead of the tight single-cursor
// tracks.
let timing = Arc::new(std::sync::Mutex::new(Vec::<f64>::new()));
// Pause/resume: `offset` is the song-second to (re)start from; `seg` records
// the wall-clock start + offset of the running segment so PAUSE can compute
// where we are. STOP resets offset to 0; PAUSE saves the current position.
let offset = Arc::new(std::sync::Mutex::new(0.0_f64));
let seg = Arc::new(std::sync::Mutex::new((Instant::now(), 0.0_f64)));
let start_perf = {
let plans = plans.clone();
let cua = cua.clone();
let playing = playing.clone();
let generation = generation.clone();
let timing = timing.clone();
let seg = seg.clone();
move |off: f64| {
playing.store(true, Ordering::SeqCst);
let g = generation.fetch_add(1, Ordering::SeqCst) + 1;
let start = Instant::now();
*seg.lock().unwrap() = (start, off);
timing.lock().unwrap().clear();
eprintln!(
"[orch] ▶ performing from {off:.1}s — {} cursors actuating in the background",
plans.len()
);
for ti in 0..plans.len() {
let plans = plans.clone();
let cua = cua.clone();
let playing = playing.clone();
let generation = generation.clone();
let timing = timing.clone();
thread::spawn(move || {
let p = &plans[ti];
let hwnd = HWND(p.hwnd_addr as *mut core::ffi::c_void);
// Thread-local adaptive lead (ms) for THIS voice only,
// warm-started at the ~constant dispatch overhead on top of
// the glide (IPC + click-post + the arrival frame) so even
// the first notes land near the beat; the EMA refines it.
let mut corr = 110.0_f64;
// One persistent daemon connection for this whole track —
// pipelines every click with no per-note process spawn (that
// spawn was the entire timing-jitter source). The daemon runs
// a handler task per connection, so the 6 tracks actuate
// concurrently; within a track, clicks are lock-step (the
// response marks the actuation moment we measure against).
let pipe = std::env::var("CUA_DRIVER_PIPE")
.unwrap_or_else(|_| r"\\.\pipe\cua-driver".into());
let mut conn = DaemonConn::open(&pipe);
for (idx, note) in p.notes.iter().enumerate() {
if !playing.load(Ordering::SeqCst) || generation.load(Ordering::SeqCst) != g
{
return;
}
if note.t < off {
continue;
} // already played before the resume point
// Fire early by the adaptive lead so the actuation lands
// on the note's scheduled time despite the glide latency.
// `start` represents song-time `off`, so schedule at the
// note's time minus that offset.
let eff_lead = (base_lead_ms + corr).max(0.0);
let due = start + Duration::from_secs_f64(note.t - off);
let fire = due
.checked_sub(Duration::from_secs_f64(eff_lead / 1000.0))
.unwrap_or(due);
let now = Instant::now();
if fire > now {
thread::sleep(fire - now);
}
if !playing.load(Ordering::SeqCst) || generation.load(Ordering::SeqCst) != g
{
return;
}
let (xf, yf) = target(p, idx, note.pitch);
if let Some((x, y)) = client_rel_to_local_px(hwnd, xf, yf) {
let args = format!(
r#"{{"pid":{},"window_id":{},"x":{},"y":{},"session":"{}"}}"#,
p.pid, p.hwnd_addr, x, y, p.session
);
let ok = match conn.as_mut() {
Some(c) => c.call("click", &args),
None => false,
};
if !ok {
// Reconnect once, else fall back to a one-shot call.
conn = DaemonConn::open(&pipe);
match conn.as_mut() {
Some(c) => {
let _ = c.call("click", &args);
}
None => {
let _ = run_call(&cua, "click", &args);
}
}
}
// Response received ⇒ the click landed (actuation moment).
let err_ms = (start.elapsed().as_secs_f64() - note.t) * 1000.0;
timing.lock().unwrap().push(err_ms);
// EMA drives the mean error to zero: eff_lead = base +
// corr and err = latency − eff_lead, so corr += α·err
// converges corr → latency − base. Clamp tight so a
// transient stall can't integrate the lead away. α is
// fairly high so short voices (a few-note pool member)
// converge within the song.
corr = (corr + 0.30 * err_ms).clamp(-base_lead_ms, 600.0);
}
}
});
}
// End-of-song timing report (one per performance/generation).
let timing = timing.clone();
let generation = generation.clone();
thread::spawn(move || {
thread::sleep(Duration::from_secs_f64((dur_sec - off).max(0.0) + 0.8));
if generation.load(Ordering::SeqCst) != g {
return;
}
report_timing(&timing.lock().unwrap(), base_lead_ms);
});
}
};
if auto {
thread::sleep(Duration::from_millis(800));
start_perf(0.0);
} else {
eprintln!("[orch] ready — click ▶ PLAY in the Transport window to start the performance.");
}
// React to the transport's PLAY / PAUSE / STOP (the human's one action that
// drives the whole coordinated ensemble — like the master window in the sibling
// demo). PLAY resumes from the held offset; PAUSE freezes the position; STOP
// resets to the top.
use std::io::{BufRead, BufReader};
let reader = BufReader::new(controller_out);
for line in reader.lines().map_while(Result::ok) {
// Drag-drop a .mid onto the transport → restart the whole demo with that
// track. Re-exec a fresh orchestrator (detached, outside our job object,
// so it survives our exit); it reaps our daemon/windows on startup and
// comes up READY on the dropped file (no --auto — the user presses ▶).
if let Some(path) = line.trim_end().strip_prefix("LOAD\t") {
eprintln!("[orch] ↻ restarting with {path}");
if let Ok(exe) = std::env::current_exe() {
let _ = Command::new(exe)
.arg(path)
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn();
}
break; // fall through to teardown; the new orchestrator takes over
}
match line.trim() {
// Idempotent: ignore PLAY while already performing (resume still works
// — PAUSE clears `playing` first), so a stray PLAY can't double-start.
"PLAY" if !playing.load(Ordering::SeqCst) => {
let off = *offset.lock().unwrap();
start_perf(off);
}
"PAUSE" => {
playing.store(false, Ordering::SeqCst);
let (rs, off) = *seg.lock().unwrap();
let pos = off + rs.elapsed().as_secs_f64();
*offset.lock().unwrap() = pos;
eprintln!("[orch] ⏸ paused at {pos:.1}s");
}
"STOP" => {
playing.store(false, Ordering::SeqCst);
*offset.lock().unwrap() = 0.0;
eprintln!("[orch] ■ stopped");
}
_ => {}
}
}
// Controller closed → tear everything down (job object kills the tree too).
playing.store(false, Ordering::SeqCst);
let _ = controller.kill();
for mut k in kids {
let _ = k.kill();
}
let _ = daemon.kill();
}
// ── song construction ──────────────────────────────────────────────────────────
/// True if a track name reads like a full drum kit (vs a single drum part like
/// "Kick", which stays a single pad).
fn name_is_kit(name: &str) -> bool {
let n = name.to_lowercase();
["drum kit", "drumkit", "drums", "percussion", "drum set"]
.iter()
.any(|s| n.contains(s))
}
/// Map a MIDI channel-10 drum note to a 3-zone pad: 0=kick, 1=snare/clap/tom,
/// 2=hat/cymbal/perc. (General MIDI percussion key map.)
fn drum_zone(pitch: u8) -> i32 {
match pitch {
35 | 36 => 0, // bass/kick drums
37 | 38 | 39 | 40 | 41 | 43 | 45 | 47 | 48 | 50 => 1, // snare, clap, rim, toms
_ => 2, // hats, cymbals, perc
}
}
/// Infer a melodic track's minigame + synth voice from its (lower-cased) name.
/// Wave keywords (square/saw/triangle/8-bit/chip/synth/scifi/…) are matched
/// explicitly so chiptune/GM names map to a fitting voice instead of plain sine.
fn infer(name: &str) -> (Kind, &'static str) {
let n = name.to_lowercase();
let has = |k: &[&str]| k.iter().any(|s| n.contains(s));
// Single drum parts (not full kits) → one-shot pad.
if has(&["kick", "bass drum", "bd"]) {
(Kind::Pad, "kick")
} else if has(&["snare", "clap", "sd"]) {
(Kind::Pad, "snare")
} else if has(&["hi-hat", "hihat", "hat", "cymbal", "ride", "shaker", "tom "]) {
(Kind::Pad, "hat")
}
// Bass before the wave words so "Bass Guitar" is a bass, not a saw lead.
else if has(&["bass"]) {
(Kind::Keys, "saw")
}
// Explicit waveform names FIRST (so "8-Bit Triangle" is a triangle, not
// caught by the chip→square fallback below).
else if has(&["sawtooth", "saw"]) {
(Kind::Keys, "saw")
} else if has(&["triangle"]) {
(Kind::Keys, "triangle")
} else if has(&["square", "pulse"]) {
(Kind::Keys, "square")
}
// Generic chiptune → square.
else if has(&["8-bit", "8bit", "chip", "nes"]) {
(Kind::Keys, "square")
}
// Timbre families.
else if has(&[
"lead", "synth", "scifi", "sci-fi", "guitar", "trumpet", "sax", "brass", "pluck",
]) {
(Kind::Keys, "square")
} else if has(&[
"pad", "string", "choir", "organ", "ambient", "smooth", "warm",
]) {
(Kind::Keys, "triangle")
} else if has(&["arp", "bell", "key", "piano", "mallet", "celesta", "harp"]) {
(Kind::Keys, "triangle")
} else {
(Kind::Keys, "sine")
}
}
fn assign_roles(song: &mut Song) {
song.tracks.truncate(MAX_TRACKS);
for (i, t) in song.tracks.iter_mut().enumerate() {
let (kind, wave) = if t.is_drum {
(Kind::Drums, "kick")
} else {
infer(&t.name)
};
t.kind = kind;
t.wave = wave;
let (sess, hex) = SESSIONS[i % SESSIONS.len()];
t.color = hex;
t.session = sess.to_string();
t.title = format!("JUKEBOX {i:02} — {}", t.name);
// Fit the pitch strip to the track's actual range so wide melodies are
// played accurately instead of being clamped to the top key: root = the
// lowest note, span = the full range (min 12 semitones for a usable
// strip, capped at 48 = 4 octaves so the keys don't get microscopic).
let lo = t.notes.iter().map(|n| n.pitch as i32).min().unwrap_or(48);
let hi = t.notes.iter().map(|n| n.pitch as i32).max().unwrap_or(72);
t.root = lo;
t.span = (hi - lo + 1).clamp(12, 48);
}
}
fn mk_track(name: &str, notes: Vec<Note>) -> Track {
Track {
name: name.into(),
notes,
kind: Kind::Keys,
wave: "sine",
root: 48,
span: KEYS_SPAN,
color: "#888",
session: String::new(),
title: String::new(),
hwnd: HWND::default(),
pid: 0,
is_drum: false,
}
}
/// Generated 8-bar, 6-part loop at 120 bpm — a 1:1 feel-port of the HTML demo
/// song so the grid moves immediately even with no .mid file.
fn demo_song() -> Song {
let q = 0.5_f64; // quarter-note seconds at 120 bpm
let bars = 8;
let roots = [36, 36, 43, 41];
let sc = [0, 2, 4, 7, 9, 12];
let (mut bass, mut kick, mut hat, mut pad, mut arp, mut lead) =
(vec![], vec![], vec![], vec![], vec![], vec![]);
for b in 0..bars {
let o = b as f64 * q * 4.0;
let r = roots[b % 4] as u8;
bass.push(Note {
t: o,
pitch: r,
vel: 100,
});
bass.push(Note {
t: o + q,
pitch: r,
vel: 80,
});
bass.push(Note {
t: o + 2.0 * q,
pitch: r + 7,
vel: 95,
});
bass.push(Note {
t: o + 3.0 * q,
pitch: r,
vel: 80,
});
for k in 0..4 {
kick.push(Note {
t: o + k as f64 * q,
pitch: 36,
vel: 110,
});
}
for h in 0..8 {
hat.push(Note {
t: o + h as f64 * q / 2.0,
pitch: 42,
vel: if h % 2 == 0 { 80 } else { 55 },
});
}
// Triads struck simultaneously — three notes at the same instant means
// one cursor can't cover them within the 100ms glide, so the Pad track
// grows a 3-cursor same-colour pool that fans out across the strip.
for iv in [12, 16, 19] {
pad.push(Note {
t: o,
pitch: r + iv,
vel: 52,
});
}
for iv in [12, 15, 19] {
pad.push(Note {
t: o + 2.0 * q,
pitch: r + iv,
vel: 50,
});
}
for a in 0..8 {
arp.push(Note {
t: o + a as f64 * q / 2.0,
pitch: r + 24 + sc[(a + b) % 6] as u8,
vel: 70,
});
}
if b % 2 == 1 {
for l in 0..4 {
lead.push(Note {
t: o + l as f64 * q,
pitch: r + 24 + sc[(l * 2) % 6] as u8,
vel: 88,
});
}
}
}
Song {
tracks: vec![
mk_track("Bass", bass),
mk_track("Kick", kick),
mk_track("Hat", hat),
mk_track("Pad", pad),
mk_track("Arp", arp),
mk_track("Lead", lead),
],
bpm: 120,
dur_sec: bars as f64 * q * 4.0,
}
}
/// Parse a Standard MIDI File into per-track note lists. Single tempo (first
/// tempo event wins) — good enough for the demo; documented in the README.
fn load_midi(path: &str) -> Result<Song, String> {
use midly::{MetaMessage, MidiMessage, Smf, Timing, TrackEventKind};
let data = std::fs::read(path).map_err(|e| e.to_string())?;
let smf = Smf::parse(&data).map_err(|e| e.to_string())?;
let ppq = match smf.header.timing {
Timing::Metrical(t) => t.as_int() as f64,
_ => 480.0,
};
// First tempo found across all tracks (us per quarter-note).
let mut tempo = 500_000.0_f64;
'outer: for tr in &smf.tracks {
for ev in tr {
if let TrackEventKind::Meta(MetaMessage::Tempo(t)) = ev.kind {
tempo = t.as_int() as f64;
break 'outer;
}
}
}
let spt = (tempo / 1e6) / ppq; // seconds per tick
let mut tracks = Vec::new();
let mut dur = 0.0_f64;
for tr in &smf.tracks {
let mut tick = 0u64;
let mut name = String::new();
// Key by (channel, pitch) so overlapping same-pitch notes on different
// channels don't collide, and polyphonic same-pitch (rare) is handled.
let mut on: std::collections::HashMap<(u8, u8), (f64, u8)> =
std::collections::HashMap::new();
let mut notes = Vec::new();
let mut drum_notes = 0usize; // notes seen on MIDI channel 10 (index 9)
for ev in tr {
tick += ev.delta.as_int() as u64;
let now = tick as f64 * spt;
match ev.kind {
TrackEventKind::Meta(MetaMessage::TrackName(bytes)) => {
if name.is_empty() {
name = String::from_utf8_lossy(bytes).trim().to_string();
}
}
TrackEventKind::Midi {
channel,
message: MidiMessage::NoteOn { key, vel },
} => {
let ch = channel.as_int();
let pitch = key.as_int();
if vel.as_int() > 0 {
on.insert((ch, pitch), (now, vel.as_int()));
if ch == 9 {
drum_notes += 1;
}
} else if let Some((t0, v)) = on.remove(&(ch, pitch)) {
notes.push(Note {
t: t0,
pitch,
vel: v,
});
dur = dur.max(now);
}
}
TrackEventKind::Midi {
channel,
message: MidiMessage::NoteOff { key, .. },
} => {
if let Some((t0, v)) = on.remove(&(channel.as_int(), key.as_int())) {
notes.push(Note {
t: t0,
pitch: key.as_int(),
vel: v,
});
dur = dur.max(now);
}
}
_ => {}
}
}
if !notes.is_empty() {
notes.sort_by(|a, b| a.t.partial_cmp(&b.t).unwrap());
let idx = tracks.len();
let nm = if name.is_empty() {
format!("Track {}", idx + 1)
} else {
name.clone()
};
// Percussion if most notes were on channel 10, or the name is a kit.
let is_drum = drum_notes * 2 > notes.len() || name_is_kit(&nm);
let mut t = mk_track(&nm, notes);
t.is_drum = is_drum;
tracks.push(t);
}
}
if tracks.is_empty() {
return Err("no note tracks".into());
}
let bpm = (60.0 / (tempo / 1e6)).round() as u32;
Ok(Song {
tracks,
bpm,
dur_sec: dur + 0.5,
})
}
// ── cua-driver call + Win32 helpers ─────────────────────────────────────────────
/// Print the per-note timing diff: how far each actuation landed from its
/// scheduled beat (actual − scheduled, ms). `mean`→0 means the adaptive lead
/// has cancelled the systematic latency; `sd`/`p90` are the residual jitter
/// (dominated by per-note `cua-driver call` subprocess spawn).
fn report_timing(errs: &[f64], base_lead_ms: f64) {
if errs.is_empty() {
eprintln!("[timing] no notes measured");
return;
}
let n = errs.len();
let mut s = errs.to_vec();
s.sort_by(|a, b| a.partial_cmp(b).unwrap());
let mean = errs.iter().sum::<f64>() / n as f64;
let sd = (errs.iter().map(|e| (e - mean).powi(2)).sum::<f64>() / n as f64).sqrt();
let absmean = errs.iter().map(|e| e.abs()).sum::<f64>() / n as f64;
let pct = |q: f64| s[((n as f64 * q) as usize).min(n - 1)];
eprintln!(
"[timing] n={n} mean={mean:+.1}ms |mean|={absmean:.1}ms sd={sd:.1}ms \
median={:+.1}ms p10={:+.1} p90={:+.1} min={:+.1} max={:+.1}",
pct(0.5),
pct(0.1),
pct(0.9),
s[0],
s[n - 1],
);
eprintln!(
"[timing] base lead {:.0}ms, refined per-voice; + ⇒ late, − ⇒ early.",
base_lead_ms,
);
}
fn run_call(cua: &PathBuf, tool: &str, json: &str) -> bool {
Command::new(cua)
.arg("call")
.arg(tool)
.arg(json)
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
}
/// A persistent connection to the `cua-driver serve` daemon over its
/// line-delimited-JSON named pipe (`\\.\pipe\cua-driver`). The daemon handles
/// many requests per connection, so holding ONE open per track lets us pipeline
/// every note's click without paying a `cua-driver call` process spawn per note
/// — that spawn (tens of ms, high variance) was the entire timing-jitter
/// source. One connection per track also means tracks dispatch concurrently
/// server-side (a handler task per connection).
struct DaemonConn {
writer: std::fs::File,
reader: std::io::BufReader<std::fs::File>,
}
impl DaemonConn {
fn open(pipe: &str) -> Option<DaemonConn> {
use std::time::Instant as I;
let deadline = I::now() + Duration::from_secs(3);
loop {
match std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(pipe)
{
Ok(f) => {
let reader = std::io::BufReader::new(f.try_clone().ok()?);
return Some(DaemonConn { writer: f, reader });
}
Err(_) if I::now() < deadline => thread::sleep(Duration::from_millis(40)),
Err(_) => return None,
}
}
}
/// Send one `call` and block for its response line. `args` is a JSON object
/// literal. Returns whether the tool reported ok. The daemon processes one
/// request per connection at a time and replies in order, so this lockstep
/// is safe and the response marks the actuation moment.
fn call(&mut self, tool: &str, args: &str) -> bool {
use std::io::{BufRead, Write};
let line = format!(r#"{{"method":"call","name":"{tool}","args":{args}}}"#) + "\n";
if self.writer.write_all(line.as_bytes()).is_err() {
return false;
}
if self.writer.flush().is_err() {
return false;
}
let mut resp = String::new();
match self.reader.read_line(&mut resp) {
Ok(0) | Err(_) => false,
Ok(_) => resp.contains("\"ok\":true"),
}
}
}
/// Window-local (x,y) fraction the cursor should click for a note. Melodic
/// strips map pitch → key cell (fixed y). Drum pads have no pitch, so we hop the
/// click around the pad each hit (deterministic per note index) — otherwise the
/// cursor would click dead-centre every time and look static; the hop makes the
/// 100ms glide visible. Must match jukebox-app's WX0/WX1/WY0/WY1/KEYS_SPAN.
fn target(p: &Voice, idx: usize, pitch: u8) -> (f64, f64) {
match p.kind {
Kind::Keys => {
let span = p.span.max(1);
let semi = (pitch as i32 - p.root).clamp(0, span - 1);
(
WX0 + (semi as f64 + 0.5) / span as f64 * (WX1 - WX0),
(WY0 + WY1) / 2.0,
)
}
Kind::Drums => {
// Click the kick / snare / hat zone this drum note belongs to.
let z = drum_zone(pitch);
(
WX0 + (z as f64 + 0.5) / 3.0 * (WX1 - WX0),
(WY0 + WY1) / 2.0,
)
}
Kind::Pad => {
let h = (idx as u32).wrapping_mul(2_654_435_761);
let xf = 0.30 + (h % 1000) as f64 / 1000.0 * 0.40;
let yf = WY0 + 0.10 + ((h / 1000) % 1000) as f64 / 1000.0 * (WY1 - WY0 - 0.20);
(xf, yf)
}
}
}
fn work_area() -> RECT {
use windows::Win32::UI::WindowsAndMessaging::{
SystemParametersInfoW, SPI_GETWORKAREA, SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS,
};
let mut wa = RECT::default();
unsafe {
let _ = SystemParametersInfoW(
SPI_GETWORKAREA,
0,
Some(&mut wa as *mut _ as *mut core::ffi::c_void),
SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0),
);
}
if wa.right <= wa.left || wa.bottom <= wa.top {
use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};
unsafe {
wa = RECT {
left: 0,
top: 0,
right: GetSystemMetrics(SM_CXSCREEN),
bottom: GetSystemMetrics(SM_CYSCREEN),
};
}
}
wa
}
fn place(hwnd: HWND, x: i32, y: i32, w: i32, h: i32, activate: bool) {
if hwnd.0.is_null() {
return;
}
let flags = if activate {
SWP_SHOWWINDOW
} else {
SWP_NOACTIVATE | SWP_SHOWWINDOW | SWP_NOZORDER
};
unsafe {
let _ = SetWindowPos(hwnd, HWND_TOP, x, y, w, h, flags);
}
}
/// Client-relative (0..1) → the click tool's window-local screenshot-pixel space
/// (ClientToScreen minus the DWM extended-frame top-left + 1px inset). Same math
/// as the multi-cursor demo.
fn client_rel_to_local_px(hwnd: HWND, rx: f64, ry: f64) -> Option<(i32, i32)> {
unsafe {
let mut cr = RECT::default();
GetClientRect(hwnd, &mut cr).ok()?;
let mut pt = POINT {
x: (rx * (cr.right - cr.left) as f64) as i32,
y: (ry * (cr.bottom - cr.top) as f64) as i32,
};
let _ = ClientToScreen(hwnd, &mut pt);
let mut dwm = RECT::default();
if DwmGetWindowAttribute(
hwnd,
DWMWA_EXTENDED_FRAME_BOUNDS,
&mut dwm as *mut _ as *mut core::ffi::c_void,
std::mem::size_of::<RECT>() as u32,
)
.is_err()
{
return Some((pt.x, pt.y));
}
Some((pt.x - dwm.left - 1, pt.y - dwm.top - 1))
}
}
// ── window discovery by title substring (case-insensitive) ──────────────────────
struct Finder {
needle: String,
hwnd: HWND,
pid: u32,
}
unsafe extern "system" fn enum_cb(hwnd: HWND, lparam: LPARAM) -> BOOL {
let f = &mut *(lparam.0 as *mut Finder);
if !IsWindowVisible(hwnd).as_bool() {
return TRUE;
}
let mut buf = [0u16; 256];
let n = GetWindowTextW(hwnd, &mut buf);
if n > 0 {
let title = String::from_utf16_lossy(&buf[..n as usize]);
if title.to_lowercase().contains(&f.needle.to_lowercase()) {
let mut pid = 0u32;
GetWindowThreadProcessId(hwnd, Some(&mut pid));
f.hwnd = hwnd;
f.pid = pid;
return BOOL(0);
}
}
TRUE
}
fn find_window_by_title(needle: &str) -> Option<(HWND, u32)> {
let deadline = Instant::now() + Duration::from_secs(8);
loop {
let mut f = Finder {
needle: needle.to_string(),
hwnd: HWND::default(),
pid: 0,
};
unsafe {
let _ = EnumWindows(Some(enum_cb), LPARAM(&mut f as *mut _ as isize));
}
if !f.hwnd.0.is_null() {
return Some((f.hwnd, f.pid));
}
if Instant::now() > deadline {
return None;
}
thread::sleep(Duration::from_millis(300));
}
}