Skip to content

armgit / moss

Public Imported from GitHub
0

MOSS: a small, mostly 3D-printed litter-picking rover. Cleaner streets, one maker at a time.

Python

cockpit: the stack stops itself when idle ("stopping the stack to preserve battery")

metrox5f7293b1909c
diff --git a/benches/bench_moss_cockpit_cold.py b/benches/bench_moss_cockpit_cold.py
index a0d6bc5..441636d 100644
--- a/benches/bench_moss_cockpit_cold.py
+++ b/benches/bench_moss_cockpit_cold.py
@@ -61,6 +61,21 @@ Every check feeds a value we know and reads back a value in physical units
file, the stack gone and its events kept, the next run, a stack with
no registry entry) and in the demo (the scripted cycle, STOP, explore,
a goal 3 s after the stop, UTC times, the stop keeping the feed)
+ J. idle stop - the rule at every branch (off, a fresh stack counted from its start,
+ activity, before the start, at the limit, not MOSS, moss-gamepad and
+ moss-base never, no stack, another clock, a start without timezone),
+ the message, MOSS_IDLE_STOP_S (under 30 s refused), fresh telemetry,
+ the dimos run rule against STACK_RE; each activity source restarts the
+ count and names itself (a fresh commanded track, a non-zero cmd_vel,
+ explore_cmd, goal_request, episode_marker, an open episode, the pose
+ moved over 10 s, a POST, keepalive) and its no-activity twin does not
+ (a lost board's repeated line, drift at rest for 30 min); the watchdog
+ with a fake launcher (once at the limit, not twice, again for a new
+ run, never at limit 0, a foreign stack, a deaf cockpit, another
+ LCM_DEFAULT_URL, zenoh, a stale pid, a busy launcher until 120 s, a
+ failing launcher tried again, a clock step); the bus tap's loop calling
+ it at 1 Hz; the real launcher's note line; /state.json idle live and
+ demo; the demo stack stopping itself after the limit
F. hygiene - no dimos/serial/torch import, no LAN address in the server
C, D and the run-change checks of G need dimos_lcm (pure python, installed with
@@ -70,7 +85,9 @@ Run: PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. .venv/bin/python benches/bench_moss
"""
import atexit
+import contextlib
import http.client
+import io
import json
import math
import os
@@ -2477,6 +2494,701 @@ cockpit_md = (ROOT / "docs" / "cockpit.md").read_text()
check("docs/cockpit.md: a 'What it decides' section, /decisions.json, the link to imu.md",
"## What it decides" in cockpit_md and "/decisions.json" in cockpit_md and "(imu.md)" in cockpit_md)
+# ---------------------------------------------------------------------------
+print("J. the idle stop: the rule, what counts as activity, the watchdog, /state.json")
+MSG5 = "stopping the stack to preserve battery (idle 5 min)"
+J_NOW = 1_800_000_000.0 # an epoch second the rule counts from
+for f in runs.glob("*.json"):
+ f.unlink()
+real_pid = mc._dimos_run_pid
+mc._dimos_run_pid = lambda: None # this machine's own dimos processes never count as a stack
+
+
+def stack_up(ago, blueprint="moss-explore"):
+ """A running stack as stack_status gives it, started `ago` seconds before J_NOW."""
+ return {"running": True, "run_id": "r", "blueprint": blueprint, "pid": 1, "log_dir": None,
+ "started_at": _dt.datetime.fromtimestamp(J_NOW - ago, _dt.timezone.utc).isoformat()}
+
+
+def rule(stack, activity_ago, limit):
+ return mc.idle_state(stack, None if activity_ago is None else J_NOW - activity_ago, J_NOW, limit)
+
+
+r_fresh = rule(stack_up(10), None, 300)
+check("rule: a fresh MOSS stack (up 10 s, no activity yet), limit 300 s -> enabled, idle 10 s (from its start), "
+ "stop in 290 s, not due",
+ r_fresh == {"enabled": True, "idle_s": 10.0, "stop_in_s": 290.0, "limit_s": 300, "due": False}, json.dumps(r_fresh))
+r_act = rule(stack_up(1000), 5, 300)
+check("rule: activity 5 s ago on a stack up 1000 s -> idle 5 s, stop in 295 s, not due",
+ (r_act["idle_s"], r_act["stop_in_s"], r_act["due"]) == (5.0, 295.0, False), json.dumps(r_act))
+r_before = rule(stack_up(10), 500, 300)
+check("rule: activity from before the stack started never counts (500 s ago, stack up 10 s) -> idle 10 s",
+ (r_before["idle_s"], r_before["stop_in_s"]) == (10.0, 290.0), json.dumps(r_before))
+edge = [rule(stack_up(1000), a, 300) for a in (299.9, 300.0, 450.0)]
+check("rule at the limit: idle 299.9 s -> not due, stop in 0.1 s; 300 s -> due, stop in 0; 450 s -> due, "
+ "stop_in_s floored at 0",
+ [(r["due"], r["stop_in_s"]) for r in edge] == [(False, 0.1), (True, 0.0), (True, 0.0)],
+ json.dumps([(r["idle_s"], r["stop_in_s"], r["due"]) for r in edge]))
+r_off = rule(stack_up(5000), None, 0)
+check("rule off (limit 0): a MOSS stack idle 5000 s -> enabled false, idle 5000 s, no stop_in_s, never due",
+ r_off == {"enabled": False, "idle_s": 5000.0, "stop_in_s": None, "limit_s": 0, "due": False}, json.dumps(r_off))
+r_foreign = rule(stack_up(5000, "explore"), None, 300)
+check("rule: a stack that is not MOSS (blueprint 'explore', idle 5000 s) -> never due, no stop_in_s",
+ (r_foreign["due"], r_foreign["stop_in_s"], r_foreign["idle_s"]) == (False, None, 5000.0), json.dumps(r_foreign))
+r_down = rule({"running": False}, 5, 300)
+r_blank = mc.idle_state({"running": True, "blueprint": "moss-explore", "started_at": None}, None, J_NOW, 300)
+check("rule: no stack -> idle_s null, not due; a running stack with no start time and no activity -> idle_s null, "
+ "not due", (r_down["idle_s"], r_down["due"], r_blank["idle_s"], r_blank["due"]) == (None, False, None, False),
+ json.dumps([r_down, r_blank]))
+r_pad, r_base, r_nav = (rule(stack_up(5000, bp), None, 300) for bp in ("moss-gamepad", "moss-base", "moss-nav"))
+check("M4 rule: moss-gamepad (the pad on the table) and moss-base idle 5000 s -> never due, no stop_in_s: they stay "
+ "up; moss-nav -> due; the stop applies to moss-nav and moss-explore only",
+ (r_pad["due"], r_pad["stop_in_s"], r_base["due"], r_base["stop_in_s"], r_nav["due"]) == (False, None, False, None, True)
+ and mc.IDLE_STOP_BLUEPRINTS == ("moss-nav", "moss-explore"), json.dumps([r_pad, r_base, r_nav]))
+r_mono = mc.idle_state(stack_up(5000), 100.0, 130.0, 300, started=120.0)
+check("m1 rule on the cockpit's clock: started=120 (the run's start converted to the monotonic clock), activity at "
+ "100, now 130 -> idle 10 s from the start, stop in 290 s (started_at is not read then)",
+ (r_mono["idle_s"], r_mono["stop_in_s"]) == (10.0, 290.0), json.dumps(r_mono))
+naive_20 = _dt.datetime.fromtimestamp(J_NOW - 20, _dt.timezone.utc).replace(tzinfo=None).isoformat()
+tz_was = os.environ.get("TZ")
+os.environ["TZ"] = "XYZ-7" # this machine seven hours east of UTC
+time.tzset()
+try:
+ r_naive = mc.idle_state({"running": True, "blueprint": "moss-explore", "started_at": naive_20}, J_NOW - 600,
+ J_NOW, 300)
+finally:
+ if tz_was is None:
+ os.environ.pop("TZ", None)
+ else:
+ os.environ["TZ"] = tz_was
+ time.tzset()
+check("m2 a started_at with no timezone is UTC, as dimOS writes it: on a machine seven hours east of UTC, a stack "
+ "started 20 s ago -> idle 20 s (not 7 h off)", r_naive["idle_s"] == 20.0, json.dumps(r_naive))
+check("the message: 300 s -> '" + MSG5 + "', 90 s -> '(idle 1.5 min)', 30 s -> '(idle 30 s)'",
+ (mc.idle_stop_message(300), mc.idle_stop_message(90), mc.idle_stop_message(30))
+ == (MSG5, "stopping the stack to preserve battery (idle 1.5 min)",
+ "stopping the stack to preserve battery (idle 30 s)"))
+err = io.StringIO()
+with contextlib.redirect_stderr(err):
+ limits = {k: mc.idle_limit_s({} if k is None else {"MOSS_IDLE_STOP_S": k})
+ for k in (None, "0", "120", "45.5", "abc", "-5", "nan", " ", "10", "29.9", "30")}
+check("MOSS_IDLE_STOP_S: unset -> 300 s, '0' -> 0 (off), '120' -> 120, '45.5' -> 45.5, '30' -> 30; 'abc', '-5', "
+ "'nan' -> 300 and the cockpit says so; m6 '10' and '29.9' (under 30 s: a stack stopped while in use) -> 300 "
+ "and it says so; blank -> 300",
+ limits == {None: 300, "0": 0, "120": 120, "45.5": 45.5, "abc": 300, "-5": 300, "nan": 300, " ": 300,
+ "10": 300, "29.9": 300, "30": 30}
+ and err.getvalue().count("MOSS_IDLE_STOP_S=") == 5 and err.getvalue().count("is under 30 s") == 2,
+ f"{limits} {err.getvalue()!r}")
+os.environ["MOSS_IDLE_STOP_S"] = "42"
+read_at_start = mc.LiveCockpit(str(Path(TMP, "never_run.sh"))).idle_limit_s
+os.environ.pop("MOSS_IDLE_STOP_S")
+check("the cockpit reads MOSS_IDLE_STOP_S from its environment when it starts (42 -> 42 s; unset -> 300 s)",
+ read_at_start == 42 and mc.LiveCockpit(str(Path(TMP, "never_run.sh"))).idle_limit_s == 300, str(read_at_start))
+check("motors commanded: output_pct [0, 0] / None / [] -> no; [35, 0], [0, -20] -> yes",
+ [mc.motors_commanded(v) for v in ([0, 0], None, [], [35, 0], [0, -20])] == [False, False, False, True, True])
+P0 = {"x": 1.0, "y": 1.0, "yaw": 0.0}
+moves = [mc.pose_moved(P0, dict(P0, **d)) for d in ({"x": 1.019}, {"x": 1.021}, {"y": 0.98 - 0.001},
+ {"yaw": math.radians(1.9)}, {"yaw": math.radians(2.1)})]
+wrap = mc.pose_moved({"x": 0.0, "y": 0.0, "yaw": math.pi - 0.01}, {"x": 0.0, "y": 0.0, "yaw": -math.pi + 0.01})
+check("pose moved: 1.9 cm -> no, 2.1 cm -> yes, 2.1 cm sideways -> yes, 1.9 deg -> no, 2.1 deg -> yes; "
+ "+179.4 deg to -179.4 deg (1.1 deg across the wrap) -> no",
+ moves == [False, True, True, False, True] and wrap is False, f"{moves} {wrap}")
+check("M1 telemetry fresh: firmware_age_s 0.1 and 1.0 -> fresh; 1.2 and 600 (the coordinator repeating the last line "
+ "of a lost board) -> stale; no age, or null -> fresh (a commanded track is never missed for it)",
+ [mc.telemetry_fresh(t) for t in ({"firmware_age_s": 0.1}, {"firmware_age_s": 1.0}, {"firmware_age_s": 1.2},
+ {"firmware_age_s": 600.0}, {}, {"firmware_age_s": None})]
+ == [True, True, False, False, True, True])
+fly_src = (ROOT / "tools" / "moss-fly.sh").read_text()
+stack_re = re.search(r"^STACK_RE='([^']+)'", fly_src, re.M)
+ARGVS = [[b"/v/bin/python3", b"/v/bin/dimos", b"run", b"moss-dimos.moss-explore"],
+ [b"/v/bin/dimos", b"--rerun-open", b"none", b"run", b"moss-dimos.moss-gamepad", b"--daemon", b""],
+ [b"/v/bin/dimos", b"status"], [b"vim", b"/v/bin/dimos"], [b"run", b"/v/bin/dimos"],
+ [b"/v/bin/dimos", b"runs"], [b"sleep", b"600"], [b"/usr/lib/systemd/systemd", b"--user"], []]
+dimos_mine = [mc.is_dimos_run(a) for a in ARGVS]
+dimos_fly = ([re.search(stack_re.group(1), " ".join(x.decode() for x in a).strip()) is not None for a in ARGVS]
+ if stack_re else None)
+check("M3 is_dimos_run (the registry pid's /proc cmdline, read before a stop) agrees with moss-fly.sh's STACK_RE on "
+ "9 command lines: dimos run, with options before run -> yes; dimos status, an editor naming the file, run "
+ "before bin/dimos, 'runs', a sleep, systemd, an empty line -> no",
+ dimos_mine == [True, True, False, False, False, False, False, False, False] and dimos_fly == dimos_mine,
+ f"{dimos_mine} {dimos_fly}")
+
+if not (HAVE_LCM_MSGS and mc.HAVE_MSGS):
+ print(" -- skipped the bus and watchdog checks, no dimos_lcm here")
+else:
+ REG_J = Path(runs, "idle.json")
+ lcm_url_was = os.environ.pop("LCM_DEFAULT_URL", None) # the cockpit on LCM's default group
+ env_lcm = {k: v for k, v in os.environ.items() if k not in ("DIMOS_TRANSPORT", "TRANSPORT", "LCM_DEFAULT_URL")}
+ FAKE_DIMOS = str(Path(TMP, "venv", "bin", "dimos")) # only a name in a command line, never run
+
+ def stand_in(env=None, blueprint="moss-dimos.moss-explore"):
+ """A live process /proc shows as a stack: `<python> -c <sleep> <venv>/bin/dimos run <blueprint>`."""
+ p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)", FAKE_DIMOS, "run", blueprint],
+ env=env_lcm if env is None else env, stdin=subprocess.DEVNULL)
+ atexit.register(p.kill)
+ deadline = time.monotonic() + 5.0 # Popen returns before the child's argv is in /proc (vfork)
+ while time.monotonic() < deadline and not mc.is_dimos_run(mc.proc_cmdline(p.pid)):
+ time.sleep(0.01)
+ return p
+
+ STAND_IN = stand_in() # a stack on LCM, the cockpit's group
+
+ def reg_j(run_id, blueprint="moss-dimos.moss-explore", ago=1000.0, pid=None):
+ """A registry entry for a stack started `ago` seconds ago (a live stand-in process, on LCM)."""
+ if run_id is None:
+ REG_J.unlink(missing_ok=True)
+ return
+ REG_J.write_text(json.dumps({"run_id": run_id, "pid": pid or STAND_IN.pid, "blueprint": blueprint,
+ "started_at": _dt.datetime.fromtimestamp(time.time() - ago, _dt.timezone.utc)
+ .isoformat(), "log_dir": "/nowhere"}))
+
+ class FakeLauncher:
+ """Records what the watchdog asks; never runs moss-fly.sh. busy_since: when the busy run
+ began (epoch s, None = unknown); fail: an error to answer, or "raise"."""
+
+ def __init__(self):
+ self.calls, self.busy, self.busy_since, self.fail = [], False, None, None
+
+ def start(self, action, note=None):
+ self.calls.append((action, note))
+ if self.fail == "raise":
+ raise PermissionError("the state dir is not writable")
+ if self.fail:
+ return {"ok": False, "error": self.fail}
+ return {"ok": True, "sent": f"moss-fly.sh {action}", "pid": 1}
+
+ def status(self, lines=True):
+ return {"running": self.busy, "action": "check" if self.busy else None, "started_at": self.busy_since,
+ "returncode": None, "note": None}
+
+ def idle_cockpit(limit=300.0):
+ c = mc.LiveCockpit(str(Path(TMP, "never_run.sh")))
+ c.launcher = FakeLauncher()
+ c.idle_limit_s = limit
+ c.listening = True # as the bus tap sets it once subscribed
+ said = []
+ real_say = c._say
+
+ def say(level, event):
+ said.append((level, event))
+ real_say(level, event)
+ c._say = say
+ return c, said
+
+ def tele_msg(pct, age=0.1):
+ doc = {"fw": "qtpy-mdd3a-v3", "mode": 2, "bus_mV": 11800, "ina_ok": True, "output_pct": pct, "target_pct": pct}
+ if age is not None:
+ doc["firmware_age_s"] = age
+ return String(data=json.dumps(doc)).lcm_encode()
+
+ def hear(c, ago=0.0):
+ """A telemetry line, no track commanded, `ago` s old: the cockpit hears the bus, nothing happened."""
+ c.bus.ingest(mc.CHANNELS["base_telemetry"], tele_msg([0, 0]), mono=time.monotonic() - ago)
+
+ def aged(c, s):
+ """The last activity `s` seconds ago, on the monotonic clock the count runs on."""
+ c.activity_mono = time.monotonic() - s
+
+ def quiet(fn):
+ with contextlib.redirect_stdout(io.StringIO()):
+ return fn()
+
+ def warnings(said):
+ return [event for level, event in said if level == "warning"]
+
+ def resets(c, feed):
+ """Age the count by 1000 s, feed the bus, let the watchdog look: did the count restart?"""
+ aged(c, 1000.0)
+ feed()
+ t = time.monotonic()
+ c.idle_tick()
+ return c.activity_mono >= t - 1.0
+
+ def twist_msg(vx, wz):
+ return Twist(linear=Vector3(x=vx, y=0.0, z=0.0), angular=Vector3(x=0.0, y=0.0, z=wz)).lcm_encode()
+
+ def odom_msg(x, y, yaw_deg=0.0):
+ h = math.radians(yaw_deg) / 2
+ return PoseStamped(header=hdr("world"), pose=pose_of(x, y, qz=math.sin(h), qw=math.cos(h))).lcm_encode()
+
+ def marker_msg(action, name):
+ return String(data=json.dumps({"action": action, "name": name, "ts": time.time()})).lcm_encode()
+
+ reg_j("idle-run-1")
+ c, said = idle_cockpit(limit=3600.0) # the 1000 s of aging below never reach this limit
+ feed = lambda ch, raw: (lambda: c.bus.ingest(mc.CHANNELS[ch], raw)) # noqa: E731
+ tel = [resets(c, feed("base_telemetry", tele_msg(p))) for p in ([0, 0], [35, 0], [0, -20])]
+ stale_tel = resets(c, lambda: None) # the last telemetry said [0, -20]; nothing new since
+ check("telemetry output_pct [0, 0] -> no activity; [35, 0] and [0, -20] (a track commanded) -> the count restarts; "
+ "no new telemetry since -> nothing (the last message is not read twice)",
+ tel == [False, True, True] and stale_tel is False, f"{tel} {stale_tel}")
+ lost = [resets(c, feed("base_telemetry", tele_msg([40, 40], age=600.0 + k))) for k in range(5)]
+ by_age = [resets(c, feed("base_telemetry", tele_msg([40, 40], age=a))) for a in (1.0, 1.2, None)]
+ check("M1 a lost board: the coordinator repeats its last line ([40, 40], firmware_age_s 600 s and growing) at "
+ "each look -> no activity, 5 looks; the same line 1.0 s old -> the count restarts; 1.2 s -> not; no age -> "
+ "restarts", lost == [False] * 5 and by_age == [True, False, True], f"{lost} {by_age}")
+ tw = [resets(c, feed("cmd_vel", twist_msg(vx, wz))) for vx, wz in ((0.0, 0.0), (0.12, 0.0), (0.0, -0.4))]
+ stale_tw = resets(c, lambda: None) # a planner that went silent on a non-zero twist
+ check("cmd_vel (0, 0) -> no activity; (0.12 m/s, 0) and (0, -0.4 rad/s) -> the count restarts; the last non-zero "
+ "twist with nothing new since -> nothing", tw == [False, True, True] and stale_tw is False, f"{tw} {stale_tw}")
+ ex = resets(c, feed("explore_cmd", Bool(data=True).lcm_encode()))
+ done = resets(c, feed("explore_done", Bool(data=True).lcm_encode()))
+ check("explore_cmd (the order to explore) -> the count restarts; explore_done alone -> nothing",
+ ex is True and done is False, f"{ex} {done}")
+ srcs = {}
+ for name, ch, raw in (("telemetry", "base_telemetry", tele_msg([35, 0])),
+ ("cmd_vel", "cmd_vel", twist_msg(0.2, 0.0)),
+ ("explore_cmd", "explore_cmd", Bool(data=True).lcm_encode()),
+ ("goal_request", "goal_request",
+ PoseStamped(header=hdr("world"), pose=pose_of(2.0, 1.0)).lcm_encode()),
+ ("episode", "episode_marker", marker_msg("stop", "ep0"))):
+ c.activity_source = "start"
+ srcs[name] = [resets(c, feed(ch, raw)), c.activity_source]
+ check("B2 + M6 each source restarts the count and names itself in idle.last_source: telemetry, cmd_vel, "
+ "explore_cmd, goal_request (a goal on the bus) and episode (an episode_marker on the bus)",
+ srcs == {k: [True, k] for k in srcs}, json.dumps(srcs))
+ c.bus.ingest(mc.CHANNELS["episode_marker"], marker_msg("start", "arm-demo"))
+ held = []
+ for _ in range(3):
+ aged(c, 1000.0)
+ c.activity_source = "start"
+ t = time.monotonic()
+ c.idle_tick()
+ held.append([c.activity_mono >= t - 1.0, c.activity_source])
+ opened = dict(c.episode or {})
+ c.bus.ingest(mc.CHANNELS["episode_marker"], marker_msg("stop", "arm-demo"))
+ c.idle_tick() # the stop marker itself is activity, once
+ after_ep = resets(c, lambda: None)
+ check("B2 an episode opened by a marker on the bus (a script, no page polling) holds the stop: the count restarts "
+ "at each look while it is open (last_source 'episode'); its stop marker closes it, then the count runs",
+ opened.get("name") == "arm-demo" and held == [[True, "episode"]] * 3 and c.episode is None
+ and after_ep is False, json.dumps([opened, held, c.episode, after_ep]))
+
+ odom_runs = [0]
+
+ def odom_looks(poses, dt=1.0):
+ """A fresh cockpit and run; one odom per look, dt s apart on the arrival clock. The looks whose pose
+ counted as a move, and the cockpit."""
+ odom_runs[0] += 1
+ reg_j(f"idle-odom-{odom_runs[0]}")
+ co, _ = idle_cockpit(limit=1e9)
+ base, hits = time.monotonic(), []
+ for k, p in enumerate(poses):
+ co.bus.ingest(mc.CHANNELS["odom"], odom_msg(*p), mono=base + k * dt)
+ before = co.activity_mono
+ co.idle_tick()
+ if co.activity_mono != before:
+ hits.append(k)
+ return hits, co
+
+ drift, _ = odom_looks([(1.0 + 0.00025 * s, 1.0, 0.0) for s in range(1800)])
+ yaw_drift, _ = odom_looks([(1.0, 1.0, 0.01 * s) for s in range(1800)])
+ check("M6 odometry drifting at rest for 30 min, one pose per look: 0.25 mm/s (45 cm in all) and 0.01 deg/s (18 deg "
+ "in all) -> never a move (the pose against the pose 10 s before: 2.5 mm, 0.1 deg)",
+ drift == [] and yaw_drift == [], f"{drift[:8]} {yaw_drift[:8]}")
+ rest = [(1.0, 1.0, 0.0)] * 12
+ push, cp = odom_looks(rest + [(1.05, 1.0, 0.0)] * 15)
+ check("M6 pushed by hand 5 cm between two looks -> a move at that look (last_source 'odom'), and at each look while "
+ "the pose 10 s before is the one before the push (looks 12 to 21), none after",
+ push == list(range(12, 22)) and cp.activity_source == "odom", str(push))
+ small, _ = odom_looks(rest + [(1.015, 1.0, 0.0)] * 5 + [(1.015, 1.0, 1.5)] * 5)
+ turn, _ = odom_looks(rest + [(1.0, 1.0, 2.5)] * 3)
+ check("M6 1.5 cm then 1.5 deg within 10 s -> no move; turned 2.5 deg between two looks -> a move (looks 12 to 14)",
+ small == [] and turn == [12, 13, 14], f"{small} {turn}")
+ reg_j("idle-run-1")
+ resets(c, feed("odom", odom_msg(1.0, 1.0))) # the run's first pose
+ reg_j("idle-run-1b")
+ od_new = resets(c, feed("odom", odom_msg(5.0, -3.0, 90.0)))
+ check("a new run (its odometry in a new frame): its first pose starts the window again, not a 5 m move",
+ od_new is False, str(od_new))
+
+ srv_j = mc.CockpitServer(("127.0.0.1", 0), c, page_dir)
+ threading.Thread(target=srv_j.serve_forever, daemon=True).start()
+ cj = Client(srv_j.server_address[1])
+ aged(c, 1000.0)
+ cj.json("/state.json")
+ kept_get = c.activity_mono < time.monotonic() - 900
+ cj.post("/cmd/stack", {"action": "check"}, header=False)
+ kept_403 = c.activity_mono < time.monotonic() - 900
+ t = time.monotonic()
+ code_ck, ans_ck = cj.post("/cmd/stack", {"action": "check"})
+ by_post = c.activity_mono >= t - 1.0 and c.activity_source == "post"
+ check("a POST /cmd/* from the page (stack check) restarts the count (last_source 'post'); a GET, or a POST refused "
+ "403 for want of the page's header, does not", by_post and code_ck == 200 and kept_get and kept_403,
+ f"post {by_post} {code_ck} get kept {kept_get} 403 kept {kept_403}")
+ c.idle_limit_s = 300.0
+ hear(c)
+ aged(c, 250.0)
+ _, before_ka = cj.json("/state.json")
+ code_ka403, _ = cj.post("/cmd/keepalive", {}, header=False)
+ kept_ka = c.activity_mono < time.monotonic() - 200
+ t = time.monotonic()
+ code_ka, ans_ka = cj.post("/cmd/keepalive", {})
+ _, after_ka = cj.json("/state.json")
+ ika = (ans_ka or {}).get("idle") or {}
+ check("M5 POST /cmd/keepalive (the idle line's button): idle 250 s -> the answer's idle restarts under 1 s with "
+ "last_source 'keepalive', /state.json says the same; without the page's header -> 403, the count kept",
+ code_ka403 == 403 and kept_ka and code_ka == 200 and ans_ka.get("ok") is True and ika.get("enabled") is True
+ and isinstance(ika.get("idle_s"), float) and ika["idle_s"] < 1.0 and ika.get("last_source") == "keepalive"
+ and (before_ka.get("idle") or {}).get("idle_s", 0) >= 249
+ and (after_ka.get("idle") or {}).get("last_source") == "keepalive" and c.activity_mono >= t - 1.0,
+ json.dumps([code_ka403, code_ka, ans_ka, before_ka.get("idle"), after_ka.get("idle")])[:500])
+
+ # the watchdog
+ c.launcher.calls.clear()
+ reg_j("idle-run-2", ago=1000.0)
+ hear(c)
+ aged(c, 299.0)
+ r0 = c.idle_tick()
+ calls0 = list(c.launcher.calls)
+ said.clear()
+ aged(c, 301.0)
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out):
+ r1 = c.idle_tick()
+ check("watchdog, limit 300 s: idle 299 s -> nothing; idle 301 s -> one stop: the launcher's 'stop' (the page's "
+ "Stop stack path) with the note '" + MSG5 + "', _say('info', the same), the line in the cockpit's own "
+ "log (its standard output, the service's journal)",
+ r0 is None and calls0 == [] and r1 is not None and r1.get("ok") and c.launcher.calls == [("stop", MSG5)]
+ and said == [("info", MSG5)] and f"moss_cockpit: info: {MSG5}" in out.getvalue(),
+ f"{r0} {calls0} {r1} {c.launcher.calls} {said} {out.getvalue()!r}")
+ again = quiet(lambda: [c.idle_tick() for _ in range(3)])
+ check("the same run, still up (going down) -> no second stop", again == [None] * 3 and len(c.launcher.calls) == 1,
+ f"{again} {c.launcher.calls}")
+ reg_j("idle-run-3", ago=400.0)
+ hear(c)
+ r3, r3b = quiet(lambda: (c.idle_tick(), c.idle_tick()))
+ check("a new run idle past the limit -> stopped again, once",
+ r3 is not None and r3b is None and c.launcher.calls == [("stop", MSG5)] * 2, str(c.launcher.calls))
+ kinds = []
+ for bp in ("other-dimos.explore", "moss-dimos.moss-gamepad", "moss-dimos.moss-base", "moss-dimos.moss-nav"):
+ reg_j("idle-run-4-" + bp.rsplit(".", 1)[-1], bp, ago=5000.0)
+ c4, said4 = idle_cockpit()
+ hear(c4)
+ aged(c4, 5000.0)
+ r4 = quiet(c4.idle_tick)
+ kinds.append([r4 is not None, len(c4.launcher.calls), c4.idle_doc(mc.stack_status())["stop_in_s"]])
+ check("M4 idle 5000 s: a stack that is not MOSS ('other-dimos.explore'), moss-gamepad (the pad on the table, as "
+ "moss-teleop.sh promises) and moss-base -> never stopped, no stop_in_s (the page hides the line); "
+ "moss-nav -> stopped", kinds == [[False, 0, None]] * 3 + [[True, 1, 0.0]], json.dumps(kinds))
+ reg_j("idle-run-5", ago=5000.0)
+ c0, said0 = idle_cockpit(limit=0.0)
+ hear(c0)
+ aged(c0, 5000.0)
+ doc0 = c0.idle_doc(mc.stack_status())
+ check("limit 0 (MOSS_IDLE_STOP_S=0): a MOSS stack idle 5000 s -> never stopped; idle doc enabled false",
+ c0.idle_tick() is None and c0.launcher.calls == [] and doc0["enabled"] is False and doc0["stop_in_s"] is None,
+ json.dumps(doc0))
+ cb, saidb = idle_cockpit()
+ hear(cb)
+ aged(cb, 5000.0)
+ cb.launcher.busy = True
+ rb = cb.idle_tick()
+ cb.launcher.busy = False
+ rb2 = quiet(cb.idle_tick)
+ check("the launcher busy (a check, a launch, a stop of the operator's) -> the watchdog waits; free -> it stops",
+ rb is None and rb2 is not None and cb.launcher.calls == [("stop", MSG5)], str(cb.launcher.calls))
+ cbl, saidbl = idle_cockpit()
+ hear(cbl)
+ aged(cbl, 5000.0)
+ cbl.launcher.busy, cbl.launcher.busy_since = True, time.time() - 60.0
+ rbl1 = cbl.idle_tick()
+ cbl.launcher.busy_since = time.time() - 130.0
+ rbl2 = quiet(cbl.idle_tick)
+ rbl3 = quiet(cbl.idle_tick)
+ check("m5 a launcher busy 60 s -> the watchdog waits; busy 130 s (over 120 s: stuck) -> it stops anyway, once, and "
+ "the journal says so once", rbl1 is None and rbl2 is not None and rbl3 is None
+ and cbl.launcher.calls == [("stop", MSG5)] and len(warnings(saidbl)) == 1
+ and "has run 130 s" in warnings(saidbl)[0], f"{cbl.launcher.calls} {saidbl}")
+ cl_, saidl = idle_cockpit()
+ hear(cl_)
+ aged(cl_, 5000.0)
+ cl_.listening = False
+ docl = cl_.idle_doc(mc.stack_status())
+ check("a cockpit not listening to the bus (it cannot see the activity), a message still in its cache -> never "
+ "stops; idle doc enabled false, blind says why",
+ quiet(cl_.idle_tick) is None and cl_.launcher.calls == [] and docl["enabled"] is False
+ and "does not listen" in (docl.get("blind") or ""), json.dumps(docl))
+ cd_, saidd = idle_cockpit()
+ aged(cd_, 5000.0)
+ rd = quiet(lambda: [cd_.idle_tick() for _ in range(3)])
+ docd = cd_.idle_doc(mc.stack_status())
+ hear(cd_, ago=11.0)
+ rd11 = quiet(cd_.idle_tick)
+ hear(cd_, ago=9.0)
+ rd9 = quiet(cd_.idle_tick)
+ check("B1 a deaf cockpit (subscribed, but nothing on any tapped channel while a MOSS stack runs: another multicast "
+ "route, a firewall) never stops it: 3 looks idle 5000 s -> nothing, idle doc enabled false with blind "
+ "'nothing heard on the bus for 10 s', the journal says it once; the last message 11 s old -> still deaf; "
+ "9 s old -> it hears, the stop goes",
+ rd == [None] * 3 and docd["enabled"] is False and "nothing heard on the bus for 10 s" in (docd.get("blind") or "")
+ and len(warnings(saidd)) == 1 and "nothing heard" in warnings(saidd)[0] and rd11 is None
+ and rd9 is not None and cd_.launcher.calls == [("stop", MSG5)],
+ json.dumps([rd, docd, saidd, rd11, rd9, cd_.launcher.calls])[:600])
+ url_other = "udpm://239.255.76.67:7668?ttl=0"
+ other = stand_in(env=dict(env_lcm, LCM_DEFAULT_URL=url_other))
+ reg_j("idle-run-url", ago=5000.0, pid=other.pid)
+ cu, saidu = idle_cockpit()
+ hear(cu)
+ aged(cu, 5000.0)
+ ru = quiet(cu.idle_tick)
+ docu = cu.idle_doc(mc.stack_status())
+ os.environ["LCM_DEFAULT_URL"] = url_other # the cockpit on the same group: it sees the run
+ try:
+ docu2 = cu.idle_doc(mc.stack_status())
+ ru2 = quiet(cu.idle_tick)
+ finally:
+ os.environ.pop("LCM_DEFAULT_URL", None)
+ other.kill()
+ other.wait()
+ check("B1 a MOSS stack started with another LCM_DEFAULT_URL than the cockpit's (another bus, even with a message "
+ "in the cache) -> never stopped, blind says why; the cockpit on the same URL -> it stops it",
+ ru is None and docu["enabled"] is False and "LCM_DEFAULT_URL" in (docu.get("blind") or "")
+ and docu2["enabled"] is True and ru2 is not None and cu.launcher.calls == [("stop", MSG5)],
+ json.dumps([docu, docu2, cu.launcher.calls]))
+ zen = stand_in(env=dict(env_lcm, DIMOS_TRANSPORT="zenoh"))
+ reg_j("idle-run-6", ago=5000.0, pid=zen.pid)
+ cz, saidz = idle_cockpit()
+ hear(cz)
+ aged(cz, 5000.0)
+ docz = cz.idle_doc(mc.stack_status())
+ check("a MOSS stack started with DIMOS_TRANSPORT=zenoh (its activity never reaches the LCM tap) -> never stopped; "
+ "idle doc enabled false", quiet(cz.idle_tick) is None and cz.launcher.calls == [] and docz["enabled"] is False,
+ json.dumps(docz))
+ zen.kill()
+ zen.wait()
+ sleeper = subprocess.Popen(["sleep", "600"], env=env_lcm, stdin=subprocess.DEVNULL)
+ atexit.register(sleeper.kill)
+ stale = []
+ for pid in (1, sleeper.pid):
+ reg_j("stale-run-moss-explore", ago=5000.0, pid=pid)
+ cs, saids = idle_cockpit()
+ hear(cs)
+ aged(cs, 5000.0)
+ rs = quiet(lambda: [cs.idle_tick() for _ in range(3)])
+ docs = cs.idle_doc(mc.stack_status())
+ stale.append([rs == [None] * 3, cs.launcher.calls == [], docs["enabled"] is False,
+ len(warnings(saids)) == 1 and "not a dimos run" in warnings(saids)[0]])
+ sleeper.kill()
+ sleeper.wait()
+ check("M3 a stale registry entry (never stopped: a power cut) whose pid now runs something else (pid 1, a sleep) "
+ "-> never stopped: its /proc cmdline is not a dimos run (the rule of _dimos_run_pid and STACK_RE); the "
+ "journal says it once; the idle doc enabled false", stale == [[True] * 4] * 2, json.dumps(stale))
+
+ class ClockStep:
+ """The time module with the wall clock stepped by `step` s (an NTP step); the monotonic clock untouched."""
+
+ def __init__(self, step):
+ self.step = step
+
+ def __getattr__(self, name):
+ return getattr(time, name)
+
+ def time(self):
+ return time.time() + self.step
+
+ reg_j("idle-run-clock", ago=60.0)
+ ck, _ = idle_cockpit()
+ hear(ck)
+ aged(ck, 10.0)
+ first_ck = ck.idle_tick() # the run's start goes to the monotonic clock, once
+ steps = []
+ try:
+ for step in (3600.0, -3600.0):
+ mc.time = ClockStep(step)
+ steps.append([ck.idle_tick(), ck.idle_doc(mc.stack_status())["idle_s"]])
+ finally:
+ mc.time = time
+ check("m1 the wall clock stepped +1 h, then -1 h, under a stack up 60 s with activity 10 s ago -> no stop, idle "
+ "stays about 10 s (the count and the run's start live on the monotonic clock)",
+ first_ck is None and all(r is None and 9.0 <= i <= 12.0 for r, i in steps) and ck.launcher.calls == [],
+ json.dumps(steps))
+ reg_j("idle-run-retry", ago=5000.0)
+ cr, saidr = idle_cockpit()
+ hear(cr)
+ aged(cr, 5000.0)
+ cr.launcher.fail = "raise"
+ tries = quiet(lambda: [cr.idle_tick() for _ in range(3)])
+ cr.launcher.fail = "launcher missing"
+ tries2 = quiet(cr.idle_tick)
+ marked = cr._idle_stopped
+ cr.launcher.fail = None
+ ok_r = quiet(cr.idle_tick)
+ after_r = quiet(cr.idle_tick)
+ check("m4 the launcher raising (state dir not writable) -> nothing escapes the watchdog, the run is not marked, it "
+ "is tried again at each look (3 tries), the journal once per error; answering not ok -> tried again; then "
+ "working -> one stop, marked, no more",
+ all(isinstance(x, dict) and x.get("ok") is False for x in tries) and (tries2 or {}).get("ok") is False
+ and marked is None and (ok_r or {}).get("ok") is True and after_r is None and len(cr.launcher.calls) == 5
+ and len(warnings(saidr)) == 2 and ("info", MSG5) in saidr,
+ json.dumps([tries, tries2, cr.launcher.calls, saidr], default=str)[:600])
+ reg_j(None)
+
+ # the real launcher writes the note where /launch.json reads the launch lines
+ stub_j = Path(TMP, "idle_stub.sh")
+ stub_j.write_text('echo "stub $1 ran"\n')
+ la_j = mc.Launcher(str(stub_j))
+ ans_j = la_j.start("stop", note=MSG5)
+ la_j._procs[-1][2].wait(5)
+ st_j = la_j.status()
+ lines_j = st_j["lines"]
+ check("the launcher with a note: last_launch.log reads '=== " + MSG5 + " <time> ===', then the stop's own "
+ "header, then the script; status note.text = the message",
+ ans_j["ok"] and len(lines_j) >= 3
+ and re.fullmatch(r"=== stopping the stack to preserve battery \(idle 5 min\) "
+ r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d ===", lines_j[0]) is not None
+ and lines_j[1].startswith("=== moss-fly.sh stop ") and "stub stop ran" in lines_j
+ and (st_j.get("note") or {}).get("text") == MSG5, json.dumps(st_j)[:300])
+ la_j.start("check")
+ la_j._procs[-1][2].wait(5)
+ check("the next launch, the page's -> no note", la_j.status(lines=False)["note"] is None)
+
+ # /state.json, live
+ reg_j("idle-run-7", ago=40.0)
+ c.idle_limit_s = 300.0
+ aged(c, 1000.0)
+ c.launcher = mc.Launcher(str(stub_j))
+ c.launcher.start("stop", note=MSG5)
+ c.launcher._procs[-1][2].wait(5)
+ cj.json("/state.json") # the new run: the cockpit drops the last run's messages
+ hear(c)
+ _, stj = cj.json("/state.json")
+ idj = stj.get("idle") or {}
+ check("live /state.json: idle {enabled, idle_s, stop_in_s, limit_s, last_source, blind} next to stack: enabled, "
+ "idle ~40 s (from the stack's start, last_source 'start'), stop in 300 - idle, limit 300, blind null; "
+ "launch.note.text = the message",
+ set(idj) == set(mc.IDLE_KEYS) and idj["enabled"] is True and 39.0 <= idj["idle_s"] <= 45.0
+ and abs(idj["idle_s"] + idj["stop_in_s"] - 300.0) < 0.2 and idj["limit_s"] == 300
+ and idj["last_source"] == "start" and idj["blind"] is None
+ and ((stj.get("launch") or {}).get("note") or {}).get("text") == MSG5,
+ json.dumps({"idle": idj, "launch": stj.get("launch")}))
+ reg_j(None)
+ _, stn = cj.json("/state.json")
+ stn_idle = stn.get("idle") or {}
+ check("live /state.json, no stack: idle enabled, idle_s and stop_in_s null (the page hides the line), blind null",
+ {k: stn_idle.get(k) for k in ("enabled", "idle_s", "stop_in_s", "limit_s", "blind")}
+ == {"enabled": True, "idle_s": None, "stop_in_s": None, "limit_s": 300.0, "blind": None}
+ and stn_idle.get("last_source") in mc.IDLE_SOURCES, json.dumps(stn_idle))
+ srv_j.shutdown()
+ STAND_IN.kill()
+ STAND_IN.wait()
+ if lcm_url_was is not None:
+ os.environ["LCM_DEFAULT_URL"] = lcm_url_was
+mc._dimos_run_pid = real_pid
+
+
+class FakeLcmHandle:
+ """lcm.LCM for the bus tap's loop: handle_timeout returns at once (no message), and ends the
+ loop by raising SystemExit (past its `except Exception` retry) once `stop` is set."""
+ stop = False
+
+ def subscribe(self, channel, fn):
+ return channel
+
+ def unsubscribe(self, sub):
+ pass
+
+ def handle_timeout(self, ms):
+ if FakeLcmHandle.stop:
+ raise SystemExit
+ time.sleep(0.002)
+ return 0
+
+
+real_lcm = mc.lcm
+mc.lcm = types.SimpleNamespace(LCM=FakeLcmHandle)
+cw = mc.LiveCockpit(str(Path(TMP, "never_run.sh")))
+looks = []
+cw.idle_tick = lambda: looks.append(time.monotonic())
+listen_t = threading.Thread(target=cw._listen, daemon=True)
+listen_t.start()
+time.sleep(0.3)
+listening_early = cw.listening
+time.sleep(2.25)
+FakeLcmHandle.stop = True
+listen_t.join(3.0)
+mc.lcm = real_lcm
+gaps = [round(b - a, 3) for a, b in zip(looks, looks[1:])]
+check("M2 the wiring: _listen with a fake lcm whose handle_timeout returns at once -> listening true, the watchdog "
+ "called about once a second (3 looks in 2.55 s, 1.0 s apart: the 1 Hz throttle), the thread ends with the bus",
+ listening_early is True and len(looks) == 3 and all(0.95 <= g <= 1.15 for g in gaps) and not listen_t.is_alive(),
+ f"listening {listening_early}, {len(looks)} looks, gaps {gaps[:5]}")
+
+# the demo: same document; its stack stops itself on the first request past the limit
+dsrv_j, dcl_j = serve(demo=True, page_dir=page_dir)
+_, sd = dcl_j.json("/state.json")
+idd = sd.get("idle") or {}
+check("demo /state.json: idle with the same keys: enabled, limit 300, idle_s a number, stop_in_s = 300 - idle_s, "
+ "blind null",
+ set(idd) == set(mc.IDLE_KEYS) and idd["enabled"] is True and idd["limit_s"] == 300
+ and isinstance(idd["idle_s"], float) and abs(idd["idle_s"] + idd["stop_in_s"] - 300.0) < 0.2
+ and idd["blind"] is None, json.dumps(idd))
+code_dka, dka = dcl_j.post("/cmd/keepalive", {})
+check("demo POST /cmd/keepalive -> ok, its idle restarts (idle_s under 1 s), last_source 'keepalive'",
+ code_dka == 200 and dka.get("ok") is True and (dka.get("idle") or {}).get("last_source") == "keepalive"
+ and (dka.get("idle") or {}).get("idle_s", 9) < 1.0, json.dumps(dka))
+dj = dsrv_j.cockpit
+dj.idle_limit_s = 1.0
+time.sleep(1.2)
+_, sd_drive = dcl_j.json("/state.json")
+check("demo, limit 1 s: the demo rover exploring (driving) is never idle -> its stack stays up (last_source 'demo')",
+ sd_drive["stack"]["running"] is True and sd_drive["idle"]["idle_s"] < 1.0
+ and sd_drive["idle"]["last_source"] == "demo", json.dumps(sd_drive["idle"]))
+dcl_j.post("/cmd/stop", {}) # halted: the rover stands still from here
+_, sd_a = dcl_j.json("/state.json")
+time.sleep(1.3)
+_, sd_b = dcl_j.json("/state.json")
+_, la_d = dcl_j.json("/launch.json")
+_, lg_d = dcl_j.json("/log.json?n=10")
+MSG1 = "stopping the stack to preserve battery (idle 1 s)"
+note_lines = [ln for ln in la_d["lines"] if MSG1 in ln]
+check("demo, halted, 1.3 s later -> the stack stopped itself: running false, launch.note.text = '" + MSG1 + "', "
+ "/launch.json the line '=== <msg> <time> === (demo)' then the stop's header, /log.json the message",
+ sd_a["stack"]["running"] is True and sd_b["stack"] == {"running": False}
+ and ((sd_b.get("launch") or {}).get("note") or {}).get("text") == MSG1 and len(note_lines) == 1
+ and la_d["lines"].index(note_lines[0]) + 1 < len(la_d["lines"])
+ and la_d["lines"][la_d["lines"].index(note_lines[0]) + 1].startswith("=== moss-fly.sh stop ")
+ and any(r.get("event") == MSG1 for r in lg_d["lines"]),
+ json.dumps({"stack": sd_b["stack"], "note": sd_b.get("launch"), "lines": la_d["lines"][-4:]})[:400])
+dcl_j.json("/state.json")
+_, la_d2 = dcl_j.json("/launch.json")
+check("demo: the same (stopped) run -> the line once", sum(MSG1 in ln for ln in la_d2["lines"]) == 1)
+dcl_j.post("/cmd/stack", {"action": "explore"}) # a new run (exploration off: nothing drives)
+_, sd_c = dcl_j.json("/state.json")
+time.sleep(1.3)
+_, sd_d = dcl_j.json("/state.json")
+_, la_d3 = dcl_j.json("/launch.json")
+check("demo: a new stack, nothing driving -> up, then stopped again after the limit (the line twice in all)",
+ sd_c["stack"]["running"] is True and (sd_c.get("launch") or {}).get("note") is None
+ and sd_d["stack"] == {"running": False} and sum(MSG1 in ln for ln in la_d3["lines"]) == 2,
+ json.dumps([sd_c["stack"].get("running"), sd_d["stack"], la_d3["lines"][-3:]])[:300])
+dsrv_j.shutdown()
+dj0 = mc.DemoCockpit()
+dj0.idle_limit_s = 0.0
+dj0.activity_mono = time.monotonic() - 5000.0
+dj0.command("stop", {})
+dj0.activity_mono = time.monotonic() - 5000.0
+sd0 = dj0.state()
+check("demo, limit 0 -> idle enabled false, the halted stack never stopped",
+ sd0["idle"]["enabled"] is False and sd0["stack"]["running"] is True, json.dumps(sd0["idle"]))
+check("the server's docstring and docs/cockpit.md name MOSS_IDLE_STOP_S and the message",
+ "MOSS_IDLE_STOP_S" in (mc.__doc__ or "") and "stopping the stack to preserve battery" in (mc.__doc__ or "")
+ and "MOSS_IDLE_STOP_S" in cockpit_md and "stopping the stack to preserve battery" in cockpit_md)
+idle_md = re.search(r"\*\*The stack stops itself\.\*\*[\s\S]*?first request past the limit\.", cockpit_md)
+idle_par = re.sub(r"\s+", " ", idle_md.group(0)) if idle_md else ""
+idle_says = ("about 2 W more than no stack at all (one measurement at the time of this change)",
+ "`moss-gamepad` and any other MOSS blueprint stay up", "for 10 s", "LCM_DEFAULT_URL", "`goal_request`",
+ "`episode_marker`", "an open episode", "POST /cmd/keepalive", "10 s before", "last_source",
+ "under 30 s is refused", "stopped at 14:05", "a restarted cockpit no longer shows it", "up to 120 s",
+ "no longer a `dimos run`", "tried again each second", "monotonic clock")
+check("docs/cockpit.md, the idle stop paragraph: the power as one measured difference (about 2 W more, no absolute "
+ "watts), moss-nav and moss-explore only (moss-gamepad stays up), the deaf cockpit (10 s) and LCM_DEFAULT_URL, "
+ "goal_request / episode_marker / an open episode, the keepalive button, the 10 s window, last_source, under "
+ "30 s refused, 'stopped at', the note lost with a restart, the 120 s launcher, the stale pid, the retry",
+ bool(idle_par) and all(s in idle_par for s in idle_says) and "6.6 W" not in cockpit_md
+ and "4.7 W" not in cockpit_md and "VDD_IN" not in idle_par,
+ ", ".join(s for s in idle_says if s not in idle_par) or idle_par[:120])
+
# ---------------------------------------------------------------------------
print("F. hygiene")
src = (ROOT / "tools" / "moss_cockpit.py").read_text()
diff --git a/benches/bench_moss_cockpit_page_cold.py b/benches/bench_moss_cockpit_page_cold.py
index c299f43..2e33b6d 100644
--- a/benches/bench_moss_cockpit_page_cold.py
+++ b/benches/bench_moss_cockpit_page_cold.py
@@ -30,6 +30,12 @@ rows, colours and words for the panel. Groups:
everything)
E. the server - when the cockpit serves /decisions.json (demo mode), its answer goes through
the page's functions: one row per event, every kind and word known
+ The idle stop's line (A to E): #idle-note under the chip, a button, hidden at first and
+ dimmed offline; dim, plain or warning colour by tokens; drawn from /state.json "idle" each
+ state, a press posting keepalive; idleView in node (m:ss, the colour by the time left, hidden
+ without a stack the stop applies to or with the rule off), renderIdle on a stub document
+ (disabled at the stop and offline), keepUp showing the count restarting, followLaunch saying
+ the cockpit's note once with its time; the demo server's idle document through idleView
Nothing here needs dimOS, a bus or hardware; D needs node (skipped with a line without it).
@@ -168,6 +174,14 @@ check(f"every id app.js reads ($, setText, meter: {len(read_ids)}) exists in ind
", ".join(missing))
check("the map caption's subtitle still carries the command (cap-sub, last_cmd_vel)",
'setText("cap-sub", sub);' in js and 'sub += (sub ? " \u00b7 " : "") + "cmd "' in js and 'id="cap-sub"' in html)
+idle_el = re.search(r'<span id="chip" [^>]*>IDLE</span>\s*</div>\s*<button id="idle-note" class="num dim" '
+ r'type="button" data-live hidden\s+title="[^"]+"></button>', html)
+check("M5 the idle line: #idle-note, a button right under the chip's row in the header (a press keeps the stack up, "
+ "its title says so), hidden until a stack the stop applies to runs, dimmed offline like the other live "
+ "values (data-live)",
+ idle_el is not None and E.get("idle-note", {}).get("tag") == "button"
+ and "Keep the stack up" in E.get("idle-note", {}).get("attrs", {}).get("title", ""),
+ json.dumps(E.get("idle-note")))
# ---------------------------------------------------------------------------
print("B. the colours")
@@ -195,6 +209,13 @@ check("amber is one token: --ui-caution in tokens.css = the depth camera's dots
"the HUD's camera line uses it", rgb is not None and depth is not None
and rgb == tuple(int(g) for g in depth.groups()) and rgb == (255, 193, 94)
and "color: var(--ui-caution)" in rule("#hud-cam"), f"{rgb} {depth.groups() if depth else None}")
+check("the idle line's colours are the page's tokens: small text, .dim = --ui-dim, .attn = --ui-attention (the warning)",
+ "font-size: var(--ui-fs-small)" in rule("#idle-note") and re.search(r"^\s*\.dim \{ color: var\(--ui-dim\); \}", html, re.M)
+ and re.search(r"^\s*\.attn \{ color: var\(--ui-attention\); \}", html, re.M) is not None, rule("#idle-note"))
+check("M5 the idle button reads as the line it was (no border, no background, no padding), mint under the pointer "
+ "while it can be pressed",
+ all(s in rule("#idle-note") for s in ("background: none", "border: 0", "padding: 0"))
+ and "color: var(--moss-mint)" in rule("#idle-note:hover:not(:disabled)"), rule("#idle-note"))
# ---------------------------------------------------------------------------
print("C. the wiring")
@@ -223,6 +244,15 @@ check(f"app.js talks to no other host: no '://', no '//host' literal, every URL
and "new WebSocket" not in js and "EventSource" not in js, ", ".join(u for u in urls if not u.startswith("/")))
check("index.html loads nothing from another host", "://" not in re.sub(r"<svg[\s\S]*?</svg>", "", html)
and all(u.startswith("/") for u in re.findall(r'(?:src|href)="([^"]+)"', html)))
+check("each state draws the idle line (render -> renderIdle) and follows the launcher first "
+ "(pollState: followLaunch(d.launch, d.stack) before render), no new endpoint",
+ re.search(r"function render\(\) \{[^}]*renderIdle\(\);", js) is not None
+ and re.search(r"followLaunch\(d\.launch, d\.stack\);\n\s*render\(\);", js) is not None
+ and "/idle" not in js)
+check("M5 a press on the idle line posts keepalive (the page's post(), with its header) and draws the answer's idle",
+ '$("idle-note").addEventListener("click", keepUp);' in js
+ and re.search(r'function keepUp\(\) \{\s*return post\("keepalive", \{\}\)', js) is not None
+ and "S.state.idle = d.idle; renderIdle();" in js)
# ---------------------------------------------------------------------------
print("D. in node")
@@ -667,6 +697,119 @@ pollDecisions().then(function () {
and idl.get("word") == "IDLE" and idl.get("cls") == "dim" and idl.get("detail") == "No stack running."
and "No stack, no decisions." in idl.get("feed", ""), json.dumps([mi, idl])[:300])
+ # The idle line and the launcher's note: idleView, renderIdle and followLaunch, the page's own code.
+ IDLE_FNS = ("pad2", "clock", "IDLE_WARN_S", "$", "setText", "stack", "idleView", "renderIdle", "followLaunch",
+ "hhmm", "keepUp", "answer")
+ idle_defs = {n: js_def(n) for n in IDLE_FNS}
+ idle_js = "\n".join(idle_defs.values()) + r"""
+var els = { "idle-note": { textContent: "", className: "num dim", hidden: true, disabled: false } };
+var document = { getElementById: function (id) { return els[id] || null; } };
+var S = { state: null, noteTs: undefined, launchBusy: false, online: true };
+var says = [], watched = 0, posted = [];
+function say(t, bad) { says.push([t, !!bad]); }
+function watchLaunch() { watched++; }
+function post(name, body) {
+ posted.push(name);
+ return Promise.resolve({ ok: true, idle: { enabled: true, idle_s: 0.2, stop_in_s: 299.8, limit_s: 300,
+ last_source: "keepalive", blind: null } });
+}
+var MOSS = { running: true, blueprint: "moss-explore" };
+function I(idle, left, lim) { return { enabled: true, idle_s: idle, stop_in_s: left, limit_s: lim === undefined ? 300 : lim }; }
+function drawn(state) {
+ S.state = state; renderIdle(); var e = els["idle-note"];
+ return { text: e.textContent, cls: e.className, hidden: e.hidden, disabled: e.disabled };
+}
+var out = {
+ sample: idleView(I(200, 100), MOSS),
+ early: idleView(I(20, 280), MOSS),
+ close: idleView(I(250, 50), MOSS),
+ due: idleView(I(300, 0), MOSS),
+ hour: idleView(I(3725, 3475, 7200), MOSS),
+ off: idleView({ enabled: false, idle_s: 50, stop_in_s: null, limit_s: 0 }, MOSS),
+ foreign: idleView({ enabled: true, idle_s: 50, stop_in_s: null, limit_s: 300 }, { running: true, blueprint: "explore" }),
+ down: idleView(I(200, 100), { running: false }),
+ none: [idleView(null, MOSS), idleView(I(200, 100), null), idleView({ enabled: true }, MOSS)],
+ shown: drawn({ stack: MOSS, idle: I(200, 100) }),
+ warn: drawn({ stack: MOSS, idle: I(250, 50) }),
+ hidden: drawn({ stack: { running: false }, idle: { enabled: true, idle_s: null, stop_in_s: null, limit_s: 300 } }),
+ dueDrawn: drawn({ stack: MOSS, idle: I(300, 0) })
+};
+S.online = false; out.offline = drawn({ stack: MOSS, idle: I(200, 100) }); S.online = true;
+var N1 = { text: "stopping the stack to preserve battery (idle 5 min)", ts: 1000 };
+var N2 = { text: "stopping the stack to preserve battery (idle 5 min)", ts: 2000 };
+// a page opened after the idle stop: the stack is down, the note says why (once)
+followLaunch({ running: false, note: N1 }, { running: false });
+followLaunch({ running: false, note: N1 }, { running: false });
+out.downFirst = { says: says.slice(), watched: watched };
+// a page opened while a later stack runs: the old note is not news; the next stop is
+says = []; watched = 0; S.noteTs = undefined;
+followLaunch({ running: false, note: N1 }, { running: true });
+out.upFirst = { says: says.slice(), watched: watched };
+followLaunch({ running: true, note: N2 }, { running: true });
+out.upNext = { says: says.slice(), watched: watched, busy: S.launchBusy };
+// a launch the page did not send, with no note (another page): followed, nothing said
+says = []; watched = 0; S.noteTs = null; S.launchBusy = false;
+followLaunch({ running: true, note: null }, { running: true });
+out.foreignLaunch = { says: says.slice(), watched: watched, busy: S.launchBusy };
+// a press on the idle line, 250 s idle: the answer's idle is drawn at once
+says = []; S.state = { stack: MOSS, idle: I(250, 50) };
+keepUp().then(function () {
+ var e = els["idle-note"];
+ out.keep = { posted: posted.slice(), says: says.slice(), text: e.textContent, cls: e.className, disabled: e.disabled };
+ console.log(JSON.stringify(out));
+});
+"""
+ r = subprocess.run([node, "-e", idle_js], capture_output=True, text=True, timeout=30)
+ try:
+ IJ = json.loads(r.stdout)
+ except ValueError:
+ IJ = {}
+ print(" node said:", r.stderr.strip()[:600])
+ check("the idle line's code is found in app.js (idleView, renderIdle, followLaunch, IDLE_WARN_S, clock)",
+ all(idle_defs.values()), ", ".join(n for n, d in idle_defs.items() if not d))
+ KEEP = " \u00b7 keep the stack up"
+ check("idle line: idle 200 s of 300 -> 'idle 3:20, stops at 5:00 to preserve battery \u00b7 keep the stack up', "
+ "plain (past half the limit), a press can keep the stack up",
+ IJ.get("sample") == {"text": "idle 3:20, stops at 5:00 to preserve battery" + KEEP, "cls": "", "keep": True},
+ json.dumps(IJ.get("sample")))
+ check("idle line: 20 s idle -> dim; 50 s left (under 60 s) -> the warning colour (attn); at the stop -> "
+ "'idle 5:00: stopping the stack to preserve battery', attn, nothing left to keep; a 2 h limit -> "
+ "'idle 1:02:05, stops at 2:00:00'",
+ (IJ.get("early") or {}).get("cls") == "dim"
+ and IJ.get("early", {}).get("text") == "idle 0:20, stops at 5:00 to preserve battery" + KEEP
+ and (IJ.get("close") or {}).get("cls") == "attn"
+ and IJ.get("due") == {"text": "idle 5:00: stopping the stack to preserve battery", "cls": "attn", "keep": False}
+ and (IJ.get("hour") or {}).get("text") == "idle 1:02:05, stops at 2:00:00 to preserve battery" + KEEP,
+ json.dumps({k: IJ.get(k) for k in ("early", "close", "due", "hour")}))
+ check("idle line hidden (null): the rule off, a stack that is not MOSS (no stop_in_s), no stack, no idle document",
+ IJ.get("off") is None and IJ.get("foreign") is None and IJ.get("down") is None
+ and IJ.get("none") == [None, None, None], json.dumps({k: IJ.get(k) for k in ("off", "foreign", "down", "none")}))
+ check("renderIdle on a stub document: shown with its text and class 'num' (+ attn under 60 s), a button that can be "
+ "pressed; hidden with no stack; disabled at the stop and while the cockpit is offline",
+ IJ.get("shown") == {"text": "idle 3:20, stops at 5:00 to preserve battery" + KEEP, "cls": "num", "hidden": False,
+ "disabled": False}
+ and IJ.get("warn") == {"text": "idle 4:10, stops at 5:00 to preserve battery" + KEEP, "cls": "num attn",
+ "hidden": False, "disabled": False}
+ and (IJ.get("hidden") or {}).get("hidden") is True and (IJ.get("dueDrawn") or {}).get("disabled") is True
+ and (IJ.get("offline") or {}).get("disabled") is True,
+ json.dumps({k: IJ.get(k) for k in ("shown", "warn", "hidden", "dueDrawn", "offline")}))
+ check("M5 keepUp (a press, 250 s idle): posts keepalive, says the count restarted, and the line shows the answer's "
+ "idle at once: 'idle 0:00, stops at 5:00', dim, still a button",
+ IJ.get("keep") == {"posted": ["keepalive"], "says": [["Idle count restarted: the stack stays up.", False]],
+ "text": "idle 0:00, stops at 5:00 to preserve battery" + KEEP, "cls": "num dim",
+ "disabled": False}, json.dumps(IJ.get("keep")))
+ msg5 = "stopping the stack to preserve battery (idle 5 min)"
+ at1, at2 = (datetime.fromtimestamp(t).strftime("%H:%M") for t in (1000, 2000))
+ check("m8 the cockpit's note on the status line with its time ('..., stopped at HH:MM', this browser's clock): a page "
+ "opened after the idle stop (no stack) says it once, in the colour that stays, and fetches the launch lines; "
+ "opened under a later stack, the old note is not said, the next stop is; a launch with no note is followed "
+ "(busy) and says nothing",
+ IJ.get("downFirst") == {"says": [[msg5 + ", stopped at " + at1, True]], "watched": 1}
+ and IJ.get("upFirst") == {"says": [], "watched": 0}
+ and IJ.get("upNext") == {"says": [[msg5 + ", stopped at " + at2, True]], "watched": 1, "busy": True}
+ and IJ.get("foreignLaunch") == {"says": [], "watched": 1, "busy": True},
+ json.dumps({k: IJ.get(k) for k in ("downFirst", "upFirst", "upNext", "foreignLaunch")}) + r.stderr[:200])
+
# ---------------------------------------------------------------------------
print("E. the server's answer through the page")
try:
@@ -716,6 +859,25 @@ console.log(JSON.stringify({ n: rows.length, labels: rows.map(function (r) { ret
elif ok_shape:
print(" -- skipped, no node here: the demo answer through the page's functions")
if server is not None:
+ with urllib.request.urlopen(base + "/state.json", timeout=10) as resp:
+ sd = json.loads(resp.read())
+ idle_doc = sd.get("idle")
+ check("demo /state.json carries idle {enabled, idle_s, stop_in_s, limit_s, last_source, blind} next to stack",
+ isinstance(idle_doc, dict)
+ and set(idle_doc) == {"enabled", "idle_s", "stop_in_s", "limit_s", "last_source", "blind"}
+ and "stack" in sd, json.dumps(idle_doc))
+ if node is not None and isinstance(idle_doc, dict):
+ idle_feed = "\n".join(js_def(n) for n in ("pad2", "clock", "IDLE_WARN_S", "idleView")) \
+ + "\nconsole.log(JSON.stringify(idleView(" + json.dumps(idle_doc) + ", " + json.dumps(sd.get("stack")) + ")));"
+ r = subprocess.run([node, "-e", idle_feed], capture_output=True, text=True, timeout=30)
+ try:
+ iv = json.loads(r.stdout)
+ except ValueError:
+ iv = None
+ check("the page draws the demo's idle document: 'idle m:ss, stops at 5:00 to preserve battery \u00b7 keep the "
+ "stack up'", isinstance(iv, dict) and re.fullmatch(r"idle \d+:\d\d, stops at 5:00 to preserve battery "
+ r"\u00b7 keep the stack up", iv.get("text", ""))
+ is not None, json.dumps(iv) + r.stderr[:200])
server.shutdown()
# The stack's own decision lines (one per kind, as a real exploration logs them) through the
diff --git a/tools/moss_cockpit.py b/tools/moss_cockpit.py
index 4915a03..06f525a 100644
--- a/tools/moss_cockpit.py
+++ b/tools/moss_cockpit.py
@@ -22,6 +22,14 @@ last_launch.log, written by the launcher it starts). It NEVER
opens a serial port: battery and arming state come from /base_telemetry, which
the stack's coordinator publishes.
+It stops an idle moss-nav or moss-explore stack by itself: no commanded track, no
+non-zero cmd_vel, no explore order, no goal, no episode, no move of the pose (2 cm or
+2 deg over 10 s) and no command from the page for MOSS_IDLE_STOP_S seconds (default
+300, at least 30, 0 = never) runs `tools/moss-fly.sh stop` with the message
+"stopping the stack to preserve battery". A cockpit that hears nothing on the bus,
+or a stack on another bus, is never stopped; moss-gamepad and the other blueprints
+stay up.
+
Endpoints (all Cache-Control: no-store; a failing request never kills the server):
GET / /static/<file> the page and its assets (tools/moss_cockpit/)
GET /state.json stack, pose, topic rates, base, nav, recording, system
@@ -46,6 +54,8 @@ Endpoints (all Cache-Control: no-store; a failing request never kills the server
explore and goal only on a moss-* stack, else 409;
bus commands only on an LCM run, else 503: the stack's own
DIMOS_TRANSPORT, then TRANSPORT, as dimOS reads them)
+ POST /cmd/keepalive the idle stop's count starts again: the stack stays up (same
+ header; the answer carries /state.json's "idle")
Standard library + numpy (+ lcm and dimos_lcm in live mode). No build step,
no CDN: the field has no internet.
@@ -163,6 +173,21 @@ LAUNCH_ENV_DROP = ("EXPLORE", "MOCK", "MOSS_MOCK_BUS") # per-run switches of a
GOAL_AFTER_STOP_S = 3.0 # the explorer's own 'stay here' goal lands ~2 s after its stop: ours goes after
MJPEG_WRITE_TIMEOUT_S = 5.0 # a viewer that vanished without FIN/RST frees its slot after this
CMD_HEADER = ("X-Requested-With", "moss-cockpit") # every POST /cmd/* carries it: a cross-site form cannot
+IDLE_STOP_DEFAULT_S = 300.0 # MOSS_IDLE_STOP_S unset: a MOSS stack idle this long is stopped (0 = never)
+IDLE_STOP_MIN_S = 30.0 # a shorter limit is refused (the default applies): it would stop a stack in use
+IDLE_STOP_BLUEPRINTS = ("moss-nav", "moss-explore") # the stacks it stops: odometry + costmap load the
+ # board; moss-gamepad and the others stay up
+IDLE_TICK_S = 1.0 # the idle watchdog looks once per second, from the bus tap's loop
+IDLE_MOVE_M = 0.02 # the pose moved more than this over IDLE_WINDOW_S: activity
+IDLE_TURN_RAD = math.radians(2.0) # ... or turned more than this
+IDLE_WINDOW_S = 10.0 # the pose now against the pose this long ago: drift at rest never adds up
+IDLE_DEAF_S = 10.0 # a MOSS stack up and nothing heard on any tapped channel this long: deaf
+IDLE_FIRMWARE_AGE_S = 1.0 # telemetry whose firmware line is older: the coordinator repeating its last one
+IDLE_BUSY_MAX_S = 120.0 # a launcher busy longer than this no longer holds the stop
+IDLE_STOP_TEXT = "stopping the stack to preserve battery"
+IDLE_KEYS = ("enabled", "idle_s", "stop_in_s", "limit_s", "last_source", "blind") # /state.json "idle"
+IDLE_SOURCES = ("start", "telemetry", "cmd_vel", "odom", "explore_cmd", "goal_request", "episode",
+ "post", "keepalive", "demo") # idle.last_source: what restarted the count last
# 3S pack: 12.6 V full, the BMS cuts at 10.2 V (linear, clamped; a voltage
# gauge, not a coulomb counter).
@@ -369,6 +394,11 @@ def decode_string(raw: bytes) -> str:
return str(LcmString.lcm_decode(raw).data)
+def decode_json(raw: bytes):
+ """A String message that carries JSON (the base telemetry), parsed."""
+ return json.loads(decode_string(raw))
+
+
# --- messages to publish ----------------------------------------------------------
# Every nested field is built fresh: the generated dimos_lcm constructors take
# their defaults as shared instances (`pose=Pose()` evaluated once), so mutating
@@ -604,6 +634,16 @@ def _parse_iso(text: str) -> float | None:
return None
+def _parse_utc(text) -> float | None:
+ """A registry time (started_at) as epoch seconds; one written without a timezone is UTC,
+ as dimOS writes it, never this machine's local time."""
+ try:
+ t = _dt.datetime.fromisoformat(str(text).replace("Z", "+00:00"))
+ except (TypeError, ValueError):
+ return None
+ return (t if t.tzinfo is not None else t.replace(tzinfo=_dt.timezone.utc)).timestamp()
+
+
def stack_status(now: float | None = None) -> dict:
"""The live run from dimOS's registry (dead pids skipped, never `dimos status`:
every dimOS CLI call writes a log file and it would also prune the registry)."""
@@ -618,7 +658,7 @@ def stack_status(now: float | None = None) -> dict:
continue
if not _pid_alive(pid):
continue
- started = _parse_iso(entry.get("started_at"))
+ started = _parse_utc(entry.get("started_at"))
if best is None or (started or 0) > (best[0] or 0):
best = (started, pid, entry)
if best is None:
@@ -632,6 +672,23 @@ def stack_status(now: float | None = None) -> dict:
"log_dir": entry.get("log_dir")}
+def proc_cmdline(pid) -> list[bytes] | None:
+ """The arguments process `pid` runs with (/proc/<pid>/cmdline), None when it is gone."""
+ try:
+ with open(f"/proc/{int(pid)}/cmdline", "rb") as fh:
+ return fh.read().split(b"\0")
+ except (OSError, TypeError, ValueError):
+ return None
+
+
+def is_dimos_run(argv) -> bool:
+ """argv (a /proc cmdline) is a `dimos run`: an argument ending in bin/dimos, then "run" among
+ the arguments after it. tools/moss-fly.sh's STACK_RE ('[b]in/dimos( [^ ]+)* run( |$)') reads
+ a command line the same way."""
+ argv = list(argv or [])
+ return any(a.endswith(b"bin/dimos") and b"run" in argv[i + 1:] for i, a in enumerate(argv))
+
+
def _dimos_run_pid() -> int | None:
"""A `dimos run` process even without a registry entry (a foreground run, a
registry swept by someone's `dimos status`). TOCTOU-safe: a process that exits
@@ -648,7 +705,7 @@ def _dimos_run_pid() -> int | None:
argv = fh.read().split(b"\0")
except OSError:
continue
- if any(a.endswith(b"bin/dimos") for a in argv) and b"run" in argv:
+ if is_dimos_run(argv):
return int(pid)
return None
@@ -669,7 +726,8 @@ def proc_environ(pid) -> dict | None:
return env
-BUS_VARS = ("DIMOS_TRANSPORT", "TRANSPORT") # what dimOS picks its bus from (explore_ctl reads them)
+BUS_VARS = ("DIMOS_TRANSPORT", "TRANSPORT", # what dimOS picks its bus from (explore_ctl reads them)
+ "LCM_DEFAULT_URL") # and the LCM group: the cockpit and the stack share all three
def bus_env(pid) -> tuple[dict, str]:
@@ -686,6 +744,107 @@ def bus_env(pid) -> tuple[dict, str]:
return {k: env[k] for k in BUS_VARS if k in env}, whose
+def bus_transport(env) -> str:
+ """The bus a stack with environment `env` runs on, in dimOS's order (explore_ctl's rule)."""
+ try:
+ import explore_ctl
+ return explore_ctl.transport(env)
+ except Exception: # noqa: BLE001 - no dimos_lcm: the same order as explore_ctl.transport
+ return (env.get("DIMOS_TRANSPORT") or env.get("TRANSPORT") or "lcm").strip().lower()
+
+
+# --- the idle stop: a stack nobody uses drains the battery ------------------------------
+#
+# With the explore stack up and the rover still, the lidar odometry, the costmap and the
+# camera keep one core of the board at 100 %. The cockpit stops a MOSS stack that saw no
+# activity for MOSS_IDLE_STOP_S seconds, as the page's Stop stack button does.
+
+def idle_limit_s(env=None) -> float:
+ """MOSS_IDLE_STOP_S from the cockpit's environment (the service's), read once at start:
+ the seconds a MOSS stack may stay idle, 0 = never stopped. Unset: the default; not a
+ number of seconds >= 0, or under IDLE_STOP_MIN_S: the default, and the cockpit says so."""
+ env = os.environ if env is None else env
+ raw = (env.get("MOSS_IDLE_STOP_S") or "").strip()
+ if not raw:
+ return IDLE_STOP_DEFAULT_S
+ try:
+ value = float(raw)
+ except ValueError:
+ value = -1.0
+ if not (math.isfinite(value) and value >= 0):
+ print(f"moss_cockpit: MOSS_IDLE_STOP_S={raw!r} is not a number of seconds >= 0: "
+ f"{IDLE_STOP_DEFAULT_S:g} s", file=sys.stderr, flush=True)
+ return IDLE_STOP_DEFAULT_S
+ if 0 < value < IDLE_STOP_MIN_S:
+ print(f"moss_cockpit: MOSS_IDLE_STOP_S={raw!r} is under {IDLE_STOP_MIN_S:g} s (0 turns the stop off): "
+ f"{IDLE_STOP_DEFAULT_S:g} s", file=sys.stderr, flush=True)
+ return IDLE_STOP_DEFAULT_S
+ return value
+
+
+def is_moss_stack(stack) -> bool:
+ """A running stack whose blueprint is a MOSS one: the only kind the cockpit commands or stops."""
+ return bool(stack and stack.get("running")) and str(stack.get("blueprint") or "").startswith("moss")
+
+
+def idle_stoppable(stack) -> bool:
+ """A running stack the idle stop may stop: moss-nav or moss-explore, whose lidar odometry and
+ costmap keep the board busy. moss-gamepad stays up with the pad on the table, as
+ tools/moss-teleop.sh promises, and so does any other blueprint."""
+ return is_moss_stack(stack) and str(stack.get("blueprint") or "") in IDLE_STOP_BLUEPRINTS
+
+
+_FROM_STACK = object() # idle_state: the start is the stack's started_at
+
+
+def idle_state(stack, activity_ts, now: float, limit_s: float, started=_FROM_STACK) -> dict:
+ """The idle rule, pure. idle_s: seconds since the last activity, or since the stack's start
+ when nothing happened since (activity before the start never counts); None with no running
+ stack or no time to count from. `started`: the stack's start on the clock of `now` and
+ `activity_ts` (the cockpit passes the monotonic clock); left out, the stack's started_at
+ as epoch seconds (UTC when it has no timezone). stop_in_s: limit_s - idle_s floored at 0,
+ for a stack the stop may stop (idle_stoppable) with the rule on, else None. due: the rule
+ is on (limit_s > 0), the stack is one of those and idle_s >= limit_s."""
+ enabled = limit_s > 0
+ running = bool(stack and stack.get("running"))
+ if started is _FROM_STACK:
+ started = _parse_utc(stack.get("started_at")) if running else None
+ marks = [t for t in (activity_ts, started)
+ if isinstance(t, (int, float)) and not isinstance(t, bool) and math.isfinite(t)]
+ idle = max(0.0, now - max(marks)) if running and marks else None
+ counted = enabled and idle is not None and idle_stoppable(stack)
+ return {"enabled": enabled, "idle_s": None if idle is None else round(idle, 1),
+ "stop_in_s": round(max(0.0, limit_s - idle), 1) if counted else None,
+ "limit_s": limit_s, "due": counted and idle >= limit_s}
+
+
+def idle_stop_message(limit_s: float) -> str:
+ """The line the cockpit says when it stops an idle stack: the limit in minutes, in seconds under one."""
+ span = f"{limit_s:g} s" if limit_s < 60 else f"{round(limit_s / 60.0, 1):g} min"
+ return f"{IDLE_STOP_TEXT} (idle {span})"
+
+
+def motors_commanded(output_pct) -> bool:
+ """Telemetry output_pct [left, right] with a track commanded (anything but [0, 0]): the pad,
+ the planner and the explorer all end there."""
+ return isinstance(output_pct, (list, tuple)) and any(
+ isinstance(v, (int, float)) and not isinstance(v, bool) and v != 0 for v in output_pct)
+
+
+def telemetry_fresh(t: dict) -> bool:
+ """A telemetry line whose firmware line is at most IDLE_FIRMWARE_AGE_S old. With the board
+ lost, the coordinator keeps publishing its last line (firmware_age_s grows): its output_pct
+ is old news. No age at all counts as fresh: a commanded track is never missed for it."""
+ age = t.get("firmware_age_s")
+ return not (isinstance(age, (int, float)) and not isinstance(age, bool) and age > IDLE_FIRMWARE_AGE_S)
+
+
+def pose_moved(ref: dict, pose: dict, move_m: float = IDLE_MOVE_M, turn_rad: float = IDLE_TURN_RAD) -> bool:
+ """The pose is more than move_m from ref, or turned more than turn_rad from it."""
+ turned = abs(math.remainder(pose["yaw"] - ref["yaw"], math.tau))
+ return math.hypot(pose["x"] - ref["x"], pose["y"] - ref["y"]) > move_m or turned > turn_rad
+
+
def log_tail(log_dir: str | None, n: int) -> list[dict]:
"""The last n records of <log_dir>/main.jsonl as {ts, level, event, logger}."""
if not log_dir:
@@ -1331,6 +1490,7 @@ class Launcher:
self._lock = threading.Lock()
self._procs: list[tuple[str, float, subprocess.Popen]] = []
self.last: dict = {}
+ self.note: dict | None = None # {text, ts}: why the cockpit itself ran the last action
def log_path(self) -> str:
return os.path.join(state_dir(), "last_launch.log")
@@ -1358,7 +1518,9 @@ class Launcher:
pass
fh.write(f"=== moss-fly.sh {action} (pid {proc.pid}) cancelled by stop: {how} ===\n".encode())
- def start(self, action: str) -> dict:
+ def start(self, action: str, note: str | None = None) -> dict:
+ """`note`: why the cockpit runs it by itself (the idle stop): a line in the log above the
+ action's header, and status()'s "note" until the next start."""
with self._lock:
busy = self._running()
if busy and action != "stop":
@@ -1372,6 +1534,8 @@ class Launcher:
with open(path, "ab" if busy else "wb") as fh:
if action == "stop":
self._cancel(fh)
+ if note:
+ fh.write(f"=== {note} {time.strftime(_ISO)} ===\n".encode())
fh.write(f"=== moss-fly.sh {action} {time.strftime(_ISO)} ===\n".encode())
fh.flush()
proc = subprocess.Popen(["bash", self.script, action], cwd=REPO_DIR, env=env,
@@ -1379,6 +1543,7 @@ class Launcher:
start_new_session=True) # its own process group: Stop can end it
self._procs = self._running() + [(action, time.time(), proc)]
self.last = {"action": action, "started_at": time.time(), "pid": proc.pid}
+ self.note = {"text": note, "ts": time.time()} if note else None
return {"ok": True, "sent": f"moss-fly.sh {action}", "pid": proc.pid}
def status(self, lines: bool = True) -> dict:
@@ -1386,8 +1551,9 @@ class Launcher:
running = self._running()
last = self._procs[-1] if self._procs else None
rc = last[2].poll() if last else None
+ note = self.note
head = {"running": bool(running), "action": last[0] if last else None,
- "started_at": last[1] if last else None, "returncode": rc}
+ "started_at": last[1] if last else None, "returncode": rc, "note": note}
if not lines:
return head
try:
@@ -1459,6 +1625,12 @@ class BusCache:
hz = (len(recent) - 1) / (recent[-1] - recent[0]) if len(recent) >= 2 and recent[-1] > recent[0] else 0.0
return {"age_s": None if mono is None else round(now - mono, 2), "hz": round(hz, 1)}
+ def last_arrival(self) -> float | None:
+ """The monotonic time of the newest message on any tapped channel, None before the first."""
+ with self._lock:
+ arrivals = [t.mono for t in self._topics.values() if t.mono is not None]
+ return max(arrivals) if arrivals else None
+
def decoded(self, name: str, fn):
"""fn(raw) for the latest message of `name`, computed once per arrival.
Returns (value, mono) or (None, None); a decode error caches None."""
@@ -1493,6 +1665,74 @@ class Cockpit:
self._cam_lock = threading.Lock()
self._cam_clients = {cam: 0 for cam in CAMERAS}
self.started = time.time()
+ self.idle_limit_s = idle_limit_s()
+ # The idle count runs on the monotonic clock: a clock step never stops a stack nor holds
+ # one. The cockpit's own start counts as activity: a cockpit restarted under a running
+ # stack gives it the whole limit, it never stops it at once for time it did not see.
+ self.activity_mono = time.monotonic()
+ self.activity_source = "start"
+ self._idle_lock = threading.Lock()
+ self._idle_stopped = None # the run the idle stop stopped: never twice
+ self._idle_start = (None, None) # (the run, its start on the monotonic clock): converted once
+ self._idle_said: set = set() # (run, what) journal lines already said for a run
+
+ def note_activity(self, source: str, mono: float | None = None) -> None:
+ """Something happened (`source`, one of IDLE_SOURCES) at `mono` (the monotonic clock, now by
+ default): the idle count restarts from there."""
+ mono = time.monotonic() if mono is None else mono
+ with self._idle_lock:
+ if mono >= self.activity_mono:
+ self.activity_mono, self.activity_source = mono, source
+
+ def _run_start_mono(self, stack: dict) -> float | None:
+ """The stack's started_at (wall clock, UTC) on the monotonic clock, converted once per run."""
+ if not (stack and stack.get("running")):
+ return None
+ key = (stack.get("run_id"), stack.get("pid"), stack.get("started_at"))
+ if self._idle_start[0] != key:
+ t = _parse_utc(stack.get("started_at"))
+ self._idle_start = (key, None if t is None else time.monotonic() - (time.time() - t))
+ return self._idle_start[1]
+
+ def _idle_rule(self, stack: dict) -> dict:
+ """idle_state on the monotonic clock, plus "source": what restarted the count last."""
+ started = self._run_start_mono(stack)
+ with self._idle_lock:
+ act, source = self.activity_mono, self.activity_source
+ st = idle_state(stack, act, time.monotonic(), self.idle_limit_s, started=started)
+ st["source"] = "start" if started is not None and started > act else source
+ return st
+
+ def _idle_blind(self, stack: dict) -> str | None:
+ """Why this cockpit cannot see the stack's activity, or None. A blind cockpit never stops a stack."""
+ return None
+
+ def idle_doc(self, stack: dict) -> dict:
+ """/state.json "idle": {enabled, idle_s, stop_in_s, limit_s, last_source, blind}. last_source:
+ what restarted the count last (IDLE_SOURCES; "start" = the stack's or the cockpit's start);
+ blind: why the cockpit cannot see the run, and then nothing is counted (enabled false)."""
+ st = self._idle_rule(stack)
+ blind = self._idle_blind(stack)
+ if blind:
+ return {"enabled": False, "idle_s": None, "stop_in_s": None, "limit_s": self.idle_limit_s,
+ "last_source": st["source"], "blind": blind}
+ return {"enabled": st["enabled"], "idle_s": st["idle_s"], "stop_in_s": st["stop_in_s"],
+ "limit_s": st["limit_s"], "last_source": st["source"], "blind": None}
+
+ def keepalive(self, stack: dict) -> dict:
+ """POST /cmd/keepalive: the operator keeps the stack up, the idle count starts again."""
+ self.note_activity("keepalive")
+ return {"ok": True, "idle": self.idle_doc(stack)}
+
+ def _say_once(self, run, what: str, level: str, event: str) -> None:
+ """A journal line said once per run: a held stop says why without repeating it every second."""
+ if (run, what) not in self._idle_said:
+ self._idle_said.add((run, what))
+ self._say(level, event)
+
+ def _say(self, level: str, event: str) -> None:
+ """The cockpit's own message line: its standard output, the service's journal."""
+ print(f"moss_cockpit: {level}: {event}", flush=True)
@property
def camera_clients(self) -> int:
@@ -1572,6 +1812,11 @@ class LiveCockpit(Cockpit):
self._goal_lock = threading.Lock()
self._goal_gen = 0 # a stop, an explore or a newer goal cancels a queued goal
self._feed = DecisionFeed() # the decisions in the running stack's main.jsonl
+ self.listening = False # the bus tap is subscribed: the idle watchdog sees the run
+ self._idle_next = 0.0 # monotonic time of the watchdog's next look
+ self._idle_seen: dict = {} # topic -> arrival of the last message the watchdog read
+ self._idle_run = None # (run_id, pid) the watchdog's pose samples belong to
+ self._idle_poses: deque = deque(maxlen=64) # (arrival, pose) per look: the last IDLE_WINDOW_S and one older
# the listener ------------------------------------------------------------
def start_listener(self) -> None:
@@ -1589,6 +1834,7 @@ class LiveCockpit(Cockpit):
lc.subscribe(ch, self._on_message)
image_subs = {cam: None for cam in CAMERAS}
self.bus_error = None
+ self.listening = True
while True:
for cam, (_, channel) in CAMERAS.items():
want = self._cam_clients[cam] > 0
@@ -1598,7 +1844,14 @@ class LiveCockpit(Cockpit):
lc.unsubscribe(image_subs[cam])
image_subs[cam] = None
lc.handle_timeout(LCM_HANDLE_MS)
+ if time.monotonic() >= self._idle_next:
+ self._idle_next = time.monotonic() + IDLE_TICK_S
+ try:
+ self.idle_tick()
+ except Exception as exc: # noqa: BLE001 - the watchdog never takes the bus tap down
+ print(f"moss_cockpit: idle watchdog: {exc!r}", file=sys.stderr, flush=True)
except Exception as exc: # noqa: BLE001 - multicast not up yet, etc.: retry
+ self.listening = False
self.bus_error = f"{type(exc).__name__}: {exc}"
time.sleep(3.0)
@@ -1617,7 +1870,7 @@ class LiveCockpit(Cockpit):
return self._pose(time.monotonic())
def _base(self, now: float) -> dict | None:
- t, mono = self.bus.decoded("base_telemetry", lambda raw: json.loads(decode_string(raw)))
+ t, mono = self.bus.decoded("base_telemetry", decode_json)
if not isinstance(t, dict):
return None
return base_view(t, now - mono)
@@ -1692,7 +1945,7 @@ class LiveCockpit(Cockpit):
"""The running stack if it is a MOSS one. Another dimOS stack may share the
computer and the bus: it never takes a goal or an explore_cmd from this page."""
st = stack_status()
- return st if st.get("running") and str(st.get("blueprint") or "").startswith("moss") else None
+ return st if is_moss_stack(st) else None
def state(self) -> dict:
now = time.monotonic()
@@ -1707,14 +1960,12 @@ class LiveCockpit(Cockpit):
if not stack["running"]:
self._close_orphan_episode()
env, _ = bus_env(stack.get("pid") if stack["running"] else None)
- try:
- import explore_ctl
- transport = explore_ctl.transport(env)
- except Exception: # noqa: BLE001 - no dimos_lcm: the same order as explore_ctl.transport
- transport = (env.get("DIMOS_TRANSPORT") or env.get("TRANSPORT") or "lcm").strip().lower()
+ transport = bus_transport(env)
+ ts = time.time()
return {
- "ts": time.time(), "mode": self.mode,
+ "ts": ts, "mode": self.mode,
"stack": stack,
+ "idle": self.idle_doc(stack),
"pose": self._pose(now),
"topics": {name: self.bus.stats(name, now) for name in STATE_TOPICS},
"base": self._base(now),
@@ -1793,6 +2044,120 @@ class LiveCockpit(Cockpit):
def preflight(self) -> dict:
return preflight_doc()
+ # the idle stop -----------------------------------------------------------
+ def _idle_blind(self, stack: dict) -> str | None:
+ if not (self.listening and HAVE_MSGS):
+ return "the cockpit does not listen to the bus"
+ if not stack.get("running"):
+ return None
+ env, _ = bus_env(stack.get("pid"))
+ if bus_transport(env) != "lcm":
+ return "the stack is not on LCM, the bus the cockpit listens to"
+ if (env.get("LCM_DEFAULT_URL") or "") != (os.environ.get("LCM_DEFAULT_URL") or ""):
+ return "the stack's LCM_DEFAULT_URL is not the cockpit's: another LCM bus"
+ if is_moss_stack(stack):
+ argv = proc_cmdline(stack.get("pid"))
+ if argv is None or not is_dimos_run(argv):
+ return f"pid {stack.get('pid')} is not a dimos run (a stale registry entry)"
+ heard = self.bus.last_arrival()
+ if heard is None or time.monotonic() - heard > IDLE_DEAF_S:
+ # every MOSS stack publishes base_telemetry at 2 Hz: silence here means this
+ # cockpit is deaf (another multicast route, a firewall), not that the rover is idle
+ return f"nothing heard on the bus for {IDLE_DEAF_S:g} s"
+ return None
+
+ def idle_tick(self) -> dict | None:
+ """The watchdog, once a second from the bus tap's loop: note what the bus shows happened
+ since its last look, then stop a moss-nav or moss-explore stack idle for idle_limit_s,
+ once per run. Returns the launcher's answer when it asked for a stop, else None."""
+ stack = stack_status()
+ self._episode_from_bus() # a marker from a script opens the episode too
+ if self.episode is not None and not stack.get("running") and _dimos_run_pid() is None:
+ self._close_orphan_episode() # no stack left to record it: it holds nothing
+ self._idle_scan(stack)
+ return self._idle_fire(stack)
+
+ def _idle_scan(self, stack: dict) -> None:
+ """Activity on the bus since the last look, each at its arrival (monotonic). Only messages
+ that arrived since then count: a planner that went silent on a non-zero twist is not driving."""
+ run = (stack.get("run_id"), stack.get("pid")) if stack.get("running") else None
+ if run != self._idle_run:
+ self._idle_run = run
+ self._idle_poses.clear() # a new run: its odometry starts a new frame
+
+ def new(name, fn):
+ value, m = self.bus.decoded(name, fn)
+ if m is None or m == self._idle_seen.get(name):
+ return None, None
+ self._idle_seen[name] = m
+ return value, m
+
+ t, at = new("base_telemetry", decode_json)
+ if isinstance(t, dict) and telemetry_fresh(t) and motors_commanded(t.get("output_pct")):
+ self.note_activity("telemetry", at)
+ tw, at = new("cmd_vel", decode_twist)
+ if tw is not None and (tw[0] != 0.0 or tw[1] != 0.0):
+ self.note_activity("cmd_vel", at)
+ order, at = new("explore_cmd", decode_bool)
+ if order:
+ self.note_activity("explore_cmd", at)
+ _, at = new("goal_request", decode_pose)
+ if at is not None:
+ self.note_activity("goal_request", at) # a goal from the page, a script or the explorer
+ _, at = new("episode_marker", decode_json)
+ if at is not None:
+ self.note_activity("episode", at)
+ if self.episode is not None:
+ self.note_activity("episode") # an open episode holds the stop
+ pose, at = new("odom", decode_pose)
+ if pose is not None and self._pose_moved(pose, at):
+ self.note_activity("odom", at)
+
+ def _pose_moved(self, pose: dict, at: float) -> bool:
+ """The pose now against the pose IDLE_WINDOW_S ago (early in a run, its first one): moved
+ more than IDLE_MOVE_M or turned more than IDLE_TURN_RAD. A window, not a sum since the last
+ move: odometry drifting a few mm a minute at rest never adds up to a move."""
+ ring = self._idle_poses
+ ring.append((at, pose))
+ while len(ring) > 1 and ring[1][0] <= at - IDLE_WINDOW_S:
+ ring.popleft()
+ return pose_moved(ring[0][1], pose)
+
+ def _idle_fire(self, stack: dict) -> dict | None:
+ """Stop a moss-nav or moss-explore stack that is due, the way the page's Stop stack button
+ does. Held while the cockpit is blind (the journal says why, once per run); tried again
+ at the next look when the launcher fails."""
+ st = self._idle_rule(stack)
+ if not st["due"]:
+ return None
+ run = (stack.get("run_id"), stack.get("pid"))
+ if run == self._idle_stopped:
+ return None
+ blind = self._idle_blind(stack)
+ if blind:
+ self._say_once(run, "blind: " + blind, "warning", f"idle {st['idle_s']:g} s, not stopped: {blind}")
+ return None
+ busy = self.launcher.status(lines=False)
+ if busy.get("running"):
+ began = busy.get("started_at")
+ age = time.time() - began if isinstance(began, (int, float)) else 0.0
+ if age <= IDLE_BUSY_MAX_S:
+ return None # a launch, a flight check or a stop is running: never over it
+ self._say_once(run, "busy", "warning", f"moss-fly.sh {busy.get('action')} has run {age:.0f} s: "
+ "the idle stop no longer waits for it")
+ msg = idle_stop_message(self.idle_limit_s)
+ try:
+ answer = self.launcher.start("stop", note=msg)
+ except Exception as exc: # noqa: BLE001 - an unwritable state dir...: tried again at the next look
+ answer = {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
+ if not answer.get("ok"):
+ self._say_once(run, f"failed: {answer.get('error')}", "warning",
+ f"{msg}: the stop did not start ({answer.get('error')}), tried again each second")
+ return answer
+ self._idle_stopped = run
+ self._say("info", msg)
+ return answer
+
# camera ------------------------------------------------------------------
def _encode(self, arr: np.ndarray, encoding: str) -> bytes:
if self._encoder is None:
@@ -1862,6 +2227,8 @@ class LiveCockpit(Cockpit):
def command(self, name: str, body: dict) -> dict:
if name == "stack":
return self._stack(body)
+ if name == "keepalive":
+ return self.keepalive(stack_status())
try:
import explore_ctl # needs dimos_lcm: imported where it is used
from dimos_lcm.std_msgs import Bool, String
@@ -2201,6 +2568,7 @@ class DemoCockpit(Cockpit):
self._log: deque = deque(maxlen=200)
self._launch_lines: deque = deque(maxlen=LAUNCH_TAIL)
self._launch_action = None
+ self._launch_note = None # {text, ts}: why the demo stopped its own stack
self._preflight_ts = now
self._decisions = DecisionLog()
self._script_t0 = None # the scripted exploration's start; None = nothing scripted runs
@@ -2278,16 +2646,42 @@ class DemoCockpit(Cockpit):
if not self._goal_said and not self._pending and self.world.goal_reached:
self._goal_said = True
self._line(now, "info", "local_planner.py", "Reached goal position, starting final rotation")
+ if self.world.moving:
+ self.note_activity("demo") # the demo rover drives: it is not idle
+ msg = self._idle_due(now)
+ if msg:
+ # the page's Stop stack path, then the note that says why
+ self.command("stack", {"action": "stop"})
+ self._launch_note = {"text": msg, "ts": now}
return now
+ def _idle_due(self, now: float) -> str | None:
+ """The idle rule on the demo stack (self._lock held): its message once per run when due.
+ The demo has no thread of its own: the stop lands on the first request past the limit."""
+ if not self._idle_rule(self._stack_doc(now))["due"]:
+ return None
+ if self._idle_stopped == self._run_id():
+ return None
+ self._idle_stopped = self._run_id()
+ msg = idle_stop_message(self.idle_limit_s)
+ self._say("info", msg)
+ self._launch_lines.append(f"=== {msg} {time.strftime(_ISO)} === (demo)")
+ return msg
+
+ def _stack_doc(self, now: float) -> dict:
+ """The demo stack as /state.json gives it."""
+ if not self.stack["running"]:
+ return {"running": False}
+ return {"running": True, "run_id": self._run_id(), "blueprint": self.stack["blueprint"], "pid": os.getpid(),
+ "started_at": _dt.datetime.fromtimestamp(self.stack["started"], _dt.timezone.utc).isoformat(),
+ "uptime_s": round(now - self.stack["started"], 1), "log_dir": None}
+
def state(self) -> dict:
now = self._tick()
w = self.world
running = self.stack["running"]
up = now - self.stack["started"]
- stack = ({"running": True, "run_id": self._run_id(), "blueprint": self.stack["blueprint"], "pid": os.getpid(),
- "started_at": _dt.datetime.fromtimestamp(self.stack["started"], _dt.timezone.utc).isoformat(),
- "uptime_s": round(up, 1), "log_dir": None} if running else {"running": False})
+ stack = self._stack_doc(now)
silent = 0.0 if running else now - (self.stack_stopped_at or now)
nominal = {"odom": 10.0, "global_costmap": 1.0, "lidar": 10.0, "pointcloud": 10.0, "path": 1.0,
"cmd_vel": 10.0, "base_telemetry": 2.0}
@@ -2305,6 +2699,7 @@ class DemoCockpit(Cockpit):
"firmware_age_s": 0.1, "mock": True, "stalled": False, "stall_count": 0}
return {
"ts": now, "mode": self.mode, "stack": stack,
+ "idle": self.idle_doc(stack),
"pose": self._pose_doc(),
"topics": topics,
"base": base_view(telemetry, 0.3) if running else None,
@@ -2317,7 +2712,7 @@ class DemoCockpit(Cockpit):
"system": system_metrics(),
"bus": {"transport": "demo", "listening": False, "error": None},
"camera": self.camera_state(),
- "launch": {"running": False, "action": self._launch_action},
+ "launch": {"running": False, "action": self._launch_action, "note": self._launch_note},
}
def _pose_doc(self) -> dict | None:
@@ -2384,7 +2779,7 @@ class DemoCockpit(Cockpit):
def launch(self) -> dict:
return {"running": False, "action": self._launch_action, "started_at": None, "returncode": 0,
- "lines": list(self._launch_lines)}
+ "note": self._launch_note, "lines": list(self._launch_lines)}
def camera_frame(self, cam: str = "nose"):
# a new key every period, so the stream really runs at 5 fps; both demo cameras
@@ -2395,6 +2790,8 @@ class DemoCockpit(Cockpit):
self._tick()
with self._lock:
w = self.world
+ if name == "keepalive":
+ return self.keepalive(self._stack_doc(time.time()))
if name == "explore":
action = body.get("action")
if action not in ("start", "stop"):
@@ -2448,6 +2845,7 @@ class DemoCockpit(Cockpit):
raise ValueError('stack needs "action": "map", "explore", "stop" or "check"')
now = time.time()
self._launch_action = action
+ self._launch_note = None
self._launch_lines.append(f"=== moss-fly.sh {action} {time.strftime(_ISO)} === (demo)")
if action != "check" and not (action in ("map", "explore") and self.stack["running"]):
self._script_t0, self._pending, self._goal_said = None, [], True
@@ -2664,6 +3062,7 @@ class Handler(BaseHTTPRequestHandler):
self._json(403, {"ok": False, "error": f"refused: POST /cmd/* needs the header "
f"'{CMD_HEADER[0]}: {CMD_HEADER[1]}'"})
return
+ self.cockpit.note_activity("post") # the operator is there: the idle count restarts
try:
length = int(self.headers.get("Content-Length") or "0")
if length < 0 or length > MAX_BODY:
@@ -2722,6 +3121,9 @@ def main(argv=None) -> int:
if not args.demo and isinstance(cockpit, LiveCockpit) and cockpit.bus_error:
print(f"warning: {cockpit.bus_error} - the page will show no live data", flush=True)
print(f"MOSS cockpit ({cockpit.mode}) on http://{args.bind}:{server.server_address[1]}/", flush=True)
+ limit = cockpit.idle_limit_s
+ print(f"idle stop: a moss-nav or moss-explore stack idle {limit:g} s is stopped" if limit > 0 else
+ "idle stop: off (MOSS_IDLE_STOP_S=0)", flush=True)
try:
server.serve_forever()
except KeyboardInterrupt:
diff --git a/tools/moss_cockpit/app.js b/tools/moss_cockpit/app.js
index 7407401..416fa06 100644
--- a/tools/moss_cockpit/app.js
+++ b/tools/moss_cockpit/app.js
@@ -8,8 +8,8 @@
* GET /map_meta 1 Hz, then /map.png only when map_meta.stamp changed
* GET /log.json 0.5 Hz GET /preflight.json on load and after a check
* GET /decisions.json?n=30 1 Hz (the stack's decisions read from its log, and the word for now)
- * GET /launch.json while a stack command runs
- * POST /cmd/<explore|goal|stop|episode|stack>
+ * GET /launch.json while a stack command runs (the page's, or one the cockpit runs itself: the idle stop)
+ * POST /cmd/<explore|goal|stop|episode|stack|keepalive> (keepalive: the idle line, a button)
*
* Each poller waits for its answer before scheduling the next one, so a slow
* Jetson never gets a queue of requests. Pollers sleep while the tab is hidden.
@@ -29,6 +29,7 @@ var CAM_GRACE_MS = 6000; // a stream younger than this is not judged dead (
var STALL_HOLD_S = 4; // the STALLED badge stays up at least this long after the stall count goes up
var DEPTH_COLOR = "rgba(255,193,94,.95)"; // amber: depth-camera obstacles, apart from the mint lidar scan
var DECISION_ROWS = 30; // the decision feed shows this many, newest on top
+var IDLE_WARN_S = 60; // the idle line under the chip turns to the warning colour this close to the stop
// The commanded-motion arrow at the robot: the path a constant (vx, wz) drives in CMD_ARROW_S,
// so 0.45 m/s is 0.6 m on the map, drawn from the marker's edge (its nose, its tail in reverse)
// and never shorter on screen than CMD_MIN_RUN; a pivot is an arc around the robot sweeping the
@@ -57,7 +58,8 @@ var S = {
view: { scale: 40, x: -3, y: 3 }, fitted: false,
pendingGoal: null, camPaused: false, camOk: false, camAt: 0, camLost: null, pfAll: false,
stateAt: 0, stallLatch: { count: null, until: 0 },
- decisions: null, decHtml: null, decKeys: null
+ decisions: null, decHtml: null, decKeys: null,
+ noteTs: undefined // the launcher note last seen (undefined: no state yet)
};
function $(id) { return document.getElementById(id); }
@@ -176,6 +178,7 @@ function pollState() {
if (wasOffline && S.offlineSince && Date.now() - S.offlineSince > RELOAD_AFTER_AWAY_MS) { location.reload(); return; }
S.state = d; S.online = true; S.stateAt = Date.now(); S.offlineSince = 0;
if (wasOffline) camRestart("Cockpit back. Reconnecting the stream."); // its stream died with it
+ followLaunch(d.launch, d.stack);
render();
}, function () {
if (S.online || !S.offlineSince) S.offlineSince = Date.now();
@@ -187,6 +190,7 @@ function pollState() {
function render() {
document.body.classList.toggle("offline", !S.online);
renderHeader();
+ renderIdle();
renderBase();
renderSystem();
renderControls();
@@ -209,6 +213,56 @@ function renderHeader() {
setText("run-name", name);
}
+// The idle stop (/state.json "idle"): how long the stack has seen no activity and when the
+// cockpit stops it, one line under the chip: {text, cls, keep}, or null to hide it (no stack the
+// stop applies to, the rule off, a cockpit that cannot see the bus). Dim until half the limit,
+// plain after, the warning colour in the last IDLE_WARN_S seconds. The line is a button: `keep`
+// while a press can still keep the stack up.
+function idleView(idle, st) {
+ if (!st || !st.running || !idle || !idle.enabled) return null;
+ var i = idle.idle_s, left = idle.stop_in_s, lim = idle.limit_s;
+ if (typeof i !== "number" || typeof left !== "number" || typeof lim !== "number") return null;
+ var cls = left < IDLE_WARN_S ? "attn" : i < lim / 2 ? "dim" : "";
+ if (left <= 0) return { text: "idle " + clock(i) + ": stopping the stack to preserve battery", cls: cls, keep: false };
+ return { text: "idle " + clock(i) + ", stops at " + clock(lim) + " to preserve battery · keep the stack up",
+ cls: cls, keep: true };
+}
+
+function renderIdle() {
+ var v = idleView(S.state && S.state.idle, stack()), el = $("idle-note");
+ el.hidden = !v;
+ if (!v) return;
+ setText("idle-note", v.text);
+ el.className = ("num " + v.cls).trim();
+ el.disabled = !v.keep || !S.online;
+}
+
+// A press on the idle line: the cockpit's count starts again (POST /cmd/keepalive) and the line
+// shows it at once, from the "idle" the answer carries.
+function keepUp() {
+ return post("keepalive", {}).then(function (d) {
+ if (!answer(d, "Idle count restarted: the stack stays up.")) return;
+ if (S.state && d.idle) { S.state.idle = d.idle; renderIdle(); }
+ }, function () { say("The cockpit did not answer.", true); });
+}
+
+// "14:05" from epoch seconds, this browser's clock
+function hhmm(ts) { var d = new Date(ts * 1000); return pad2(d.getHours()) + ":" + pad2(d.getMinutes()); }
+
+// A stack command the page did not send (the cockpit's own idle stop, another page's launch):
+// its lines go to the stack tail like the page's own, and the cockpit's note (why it ran) goes
+// to the status line with its time ("..., stopped at 14:05"), where it stays until the next action.
+// On the first state a note is said only when no stack runs: then it is why the stack is down, not
+// news about a later stack. The cockpit keeps the note in memory: a restarted cockpit has none.
+function followLaunch(l, st) {
+ var n = l && l.note, key = n && n.text ? n.ts : null;
+ var said = key !== null && key !== S.noteTs && (S.noteTs !== undefined || !(st && st.running));
+ if (said) say(String(n.text) + (typeof n.ts === "number" && isFinite(n.ts) ? ", stopped at " + hhmm(n.ts) : ""), true);
+ S.noteTs = key;
+ if (l && l.running) S.launchBusy = true;
+ if (said || (l && l.running)) watchLaunch();
+}
+
function modeName(b) {
if (b.mode_name) return b.mode_name;
return ({ 0: "off", 1: "ready", 2: "drive", 3: "test" })[b.mode] || "--";
@@ -970,6 +1024,7 @@ function watchLaunch() {
$("b-map").addEventListener("click", function () { stackCmd("map", "Launching the mapping stack."); });
$("b-explore").addEventListener("click", function () { stackCmd("explore", "Launching the explore stack. Exploration stays off."); });
$("b-stack-stop").addEventListener("click", function () { stackCmd("stop", "Stopping the stack."); });
+$("idle-note").addEventListener("click", keepUp);
$("b-check").addEventListener("click", function () { stackCmd("check", "Flight check running."); });
$("b-check2").addEventListener("click", function () { stackCmd("check", "Flight check running."); });
diff --git a/tools/moss_cockpit/index.html b/tools/moss_cockpit/index.html
index c730edd..db92b91 100644
--- a/tools/moss_cockpit/index.html
+++ b/tools/moss_cockpit/index.html
@@ -51,6 +51,11 @@
.chip.idle { color: var(--ui-unknown); }
.chip.off { color: var(--moss-salmon); }
.chip.off::before { background: transparent; box-shadow: inset 0 0 0 1.5px currentColor; }
+ /* the idle stop's count, a button that keeps the stack up; .dim far from the stop, .attn close */
+ #idle-note { font-size: var(--ui-fs-small); appearance: none; background: none; border: 0; padding: 0;
+ text-align: left; cursor: pointer; align-self: flex-start; }
+ #idle-note:hover:not(:disabled) { color: var(--moss-mint); }
+ #idle-note:disabled { cursor: default; }
@keyframes breathe { 50% { opacity: .35; } }
main { display: grid; grid-template-columns: minmax(250px, 3fr) minmax(0, 6fr) minmax(290px, 3fr);
@@ -310,6 +315,8 @@
<span id="run-name" class="num" data-live>no stack</span>
<span id="chip" class="chip idle" role="status" aria-live="polite">IDLE</span>
</div>
+ <button id="idle-note" class="num dim" type="button" data-live hidden
+ title="Keep the stack up: the idle count starts again"></button>
</div>
</header>