Skip to content

Debloat any Android or Google TV over Wi-Fi and get an Apple TV-style home.

tools: bench.py prints tools/tv.py's sentence for a TV it cannot reach

xbshengec1e7616cd50
diff --git a/docs/workflows/perf-ab.md b/docs/workflows/perf-ab.md
index ec6e89286..fda9eaa78 100644
--- a/docs/workflows/perf-ab.md
+++ b/docs/workflows/perf-ab.md
@@ -46,7 +46,8 @@ Some notes on the numbers:
- **Noise:** two runs of the same build on a TCL C755 differed by 0 ms in key response, up to 45 ms in
cold start and 1 MB in PSS, and the limits sit above that. Rerun both builds if a field lands near
its limit.
-- **Run time:** a full run takes about 6 minutes.
+- **Run time:** a full run takes about 6 minutes, and the TV has to stay awake for it: a set that goes to
+ standby drops ADB, and the run stops there without writing `result.json`.
- **TVs that don't log injected keys:** `return_to_home_ms` is `null`, and `response_ms` includes the ADB
round trip. It still compares two builds on the same TV.
diff --git a/tests/test_bench_connection.py b/tests/test_bench_connection.py
index 8021ffda9..62c2e8819 100644
--- a/tests/test_bench_connection.py
+++ b/tests/test_bench_connection.py
@@ -7,18 +7,38 @@ relay one full run took over 45 minutes. Now the TV class connects once with too
and reuses it, reopening it once if it drops; only the log's newest lines are read, filtered on the
TV, and not at all once the first keys show the TV doesn't log them; and the key loop looks up Home once. --host also has to reach the client: the old subprocesses only read TV_HOST. It reaches it as TV_HOST/TV_PORT, and the target the run keeps is the one tools/tv.py made of those two (it splits `host:port` and checks the range), not the arguments again.
-The TV and tools/tv.py are faked, so these run without either.
+A TV that cannot be reached is the other half, and the run's first connect is also the one it reopens
+after a drop: tools/tv.py words a refused port, a Wireless debugging port and an address nothing answers
+on in connect_error(), and bench.py let the exception out as a traceback with exit 1 - naming neither the
+address nor that no result.json was written, and 1 is also the status a finished run whose result.json
+could not be written returns, so nothing told the two apart. The sentence is printed now, and a failure
+tv.py has nothing to add to (a reset in the handshake, a key file it cannot read) keeps its traceback,
+which names the file.
+
+The TV and tools/tv.py are faked, so these run without either; one test runs the real pair against a port
+on 127.0.0.1 that nothing listens on.
Run from the repository root: python3 -m unittest discover -s tests -v
"""
+import contextlib
+import io
import os
+from pathlib import Path
+import shutil
+import subprocess
import sys
+import tempfile
import types
import unittest
from unittest import mock
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import test_tv_connect_errors as connect_errors # noqa: E402 (a module, so its tests do not run twice)
+
from tools import bench
+ROOT = Path(__file__).resolve().parents[1]
+
class FakeDevice:
def __init__(self, answers=None, fail_first=False):
@@ -43,6 +63,8 @@ def fake_client(devices):
HOST and PORT stand for the target the real client resolves from TV_HOST/TV_PORT when bench.py
imports it; this one has no parser, so the environment bench.py sets for it is what a test reads.
+ connect_error() is the real client's too: None for an error it has nothing to add to, which is
+ what keeps such an error's traceback.
"""
module = types.ModuleType("tv")
module.HOST, module.PORT = None, None
@@ -52,6 +74,7 @@ def fake_client(devices):
module.connects += 1
return devices[module.connects - 1]
module.connect = connect
+ setattr(module, "connect_error", lambda error: None)
return module
@@ -157,5 +180,102 @@ class KeyTest(unittest.TestCase):
self.assertIn("dumpsys gfxinfo com.gathin.tvhome reset", tv.commands)
+class UnreachableTVTest(unittest.TestCase):
+ """A TV that cannot be reached ends the run with tools/tv.py's sentence, not a traceback.
+
+ tv.py words the failures that mean the TV was not reached (a refused port, a Wireless debugging port,
+ an address nothing answers on) in connect_error(), and check.py, b64push.py and privacy.py print that
+ instead of a traceback. bench.py let the exception out: a TV that was asleep, or off, or had its
+ debugging switched off, ended a run in a ConnectionRefusedError/TimeoutError traceback with exit 1 -
+ naming neither the address nor that no file was written, and 1 is also the code a finished run whose
+ result.json could not be written returns, so a run that never reached the TV looked like one that did.
+ """
+
+ def client_that_raises(self, error, why):
+ module = types.ModuleType("tv")
+ module.HOST, module.PORT = "192.0.2.9", 5555
+ module.connects = 0
+
+ def connect():
+ module.connects += 1
+ raise error
+
+ module.connect = connect
+ setattr(module, "connect_error", lambda raised: why)
+ patcher = mock.patch.dict(sys.modules, {"tv": module})
+ patcher.start()
+ self.addCleanup(patcher.stop)
+ return module
+
+ def test_tv_pys_sentence_is_printed_and_the_run_stops_at_2(self):
+ sentence = ("could not reach 192.0.2.9:5555 (timed out). Is the TV on and on this network? "
+ "See docs/TROUBLESHOOTING.md#connecting.")
+ client = self.client_that_raises(TimeoutError("timed out"), sentence)
+ tv = bench.TV("192.0.2.9", 5555)
+ out, err = io.StringIO(), io.StringIO()
+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
+ with self.assertRaises(SystemExit) as caught:
+ tv.shell("true")
+ self.assertEqual(2, caught.exception.code)
+ self.assertIn(sentence, err.getvalue())
+ self.assertIn("Nothing was written", err.getvalue())
+ self.assertEqual("", out.getvalue())
+ # The run ends here: shell() reopens a dropped connection once, and this SystemExit is not a drop.
+ self.assertEqual(1, client.connects)
+ self.assertIsNone(tv.client)
+
+ def test_an_error_tv_py_has_nothing_to_add_to_keeps_its_traceback(self):
+ # A reset in the handshake reached the TV, and a key file tv.py cannot read carries the file's
+ # name; connect_error() returns None for both, and neither may read as "the TV was off".
+ client = self.client_that_raises(ConnectionResetError("connection reset by peer"), None)
+ tv = bench.TV("192.0.2.9", 5555)
+ with self.assertRaises(ConnectionResetError):
+ tv.shell("true")
+ # One reopen (shell()'s second attempt), then the exception: no third try.
+ self.assertEqual(2, client.connects)
+
+
+class BenchAgainstARefusingPortTest(unittest.TestCase):
+ """The real tools/tv.py and a port on 127.0.0.1 nothing listens on: what a reader actually sees.
+
+ Run from a copy of the two scripts in a temporary folder (bench.py imports tv.py from beside itself),
+ with HOME there too so the ADB key tv.py creates stays out of the checkout.
+ """
+
+ @classmethod
+ def setUpClass(cls):
+ cls.python = connect_errors.client_python()
+ if not cls.python:
+ raise unittest.SkipTest("no interpreter with adb_shell to run bench.py")
+
+ def setUp(self):
+ self._tmp = tempfile.TemporaryDirectory()
+ self.folder = Path(self._tmp.name)
+ shutil.copyfile(ROOT / "tools" / "bench.py", self.folder / "bench.py")
+ shutil.copyfile(ROOT / "tools" / "tv.py", self.folder / "tv.py")
+ self.home = self.folder / "home"
+ self.home.mkdir()
+
+ def tearDown(self):
+ self._tmp.cleanup()
+
+ def test_the_run_names_the_port_and_writes_nothing(self):
+ clean = {k: v for k, v in os.environ.items() if k not in ("TV_HOST", "TV_PORT", "PYTHONPATH")}
+ clean.update(HOME=str(self.home), USERPROFILE=str(self.home),
+ TV_HOST=f"127.0.0.1:{connect_errors.refusing_port()}")
+ result = subprocess.run([self.python, "-I", str(self.folder / "bench.py"),
+ "--out-dir", "results"],
+ cwd=str(self.folder), env=clean, capture_output=True, text=True, timeout=180)
+ self.assertEqual(2, result.returncode, result.stdout + result.stderr)
+ self.assertEqual("", result.stdout)
+ self.assertNotIn("Traceback", result.stderr)
+ self.assertIn("refused the connection: nothing listens for ADB there", result.stderr)
+ self.assertIn("docs/TROUBLESHOOTING.md#connecting", result.stderr)
+ self.assertIn("Nothing was written", result.stderr)
+ # Nothing was asked of the TV and no results folder was made for a run that did not happen.
+ self.assertFalse((self.folder / "results").exists())
+ self.assertFalse(list(self.folder.glob("**/result.json")))
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/tools/bench.py b/tools/bench.py
index 490b17dc2..19b3ef991 100644
--- a/tools/bench.py
+++ b/tools/bench.py
@@ -27,7 +27,8 @@ The output is a summary plus the raw per-key numbers, so a run can be compared w
re-analysed later.
Exit status: 0 the run is measured and written; 2 an --out-dir that cannot be written into, judged
-before the TV is opened, or a folder that cannot be created (nothing was measured); 1 anything else,
+before the TV is opened, a folder that cannot be created, or a TV that cannot be reached (tools/tv.py's
+sentence says which failure it was) - nothing was measured, and nothing was written; 1 anything else,
including a result.json that could not be written after the run, whose numbers are then printed.
Why the frame is matched by its completion time and not by its intended vsync: on a launcher
@@ -96,7 +97,25 @@ class TV:
# `:port` back into the host, so `--host 192.0.2.20:5877` asked for a machine of that name.
self.host, self.port = tv_client.HOST, tv_client.PORT
self.serial = f"{self.host}:{self.port}"
- self.client = tv_client.connect()
+ try:
+ self.client = tv_client.connect()
+ except Exception as error:
+ # tools/tv.py words the failures that mean the TV was not reached: a port that refuses,
+ # a Wireless debugging port that asks for TLS, an address nothing answers on, and the
+ # macOS Local Network denial that looks like one. The errors it has nothing to add to (a
+ # key file it cannot read, a connection reset in the handshake) keep their traceback.
+ # Left alone, a refused port here ended the run in a ConnectionRefusedError traceback
+ # with exit 1, naming neither the address nor what the run lost, and exit 1 is also what
+ # a finished run whose result.json could not be written returns - so a TV that was
+ # asleep read as a benchmark that ran. This is the one function both the first connect
+ # and the reopen after a dropped connection go through; a run that stops here has
+ # written nothing, because result.json comes at the end.
+ why = tv_client.connect_error(error)
+ if why is None:
+ raise
+ print(f"{why}\nNothing was written: result.json is written at the end of a run.",
+ file=sys.stderr)
+ raise SystemExit(2) from error
return self.client
def shell(self, command):