Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

7.3 KB
Raw Blame
#!/usr/bin/env python3
"""Tests for windows_test_baseline.py. Run: python3 desktop/tool/test_windows_test_baseline.py"""
import contextlib
import io
import json
import sys
import tempfile
import unittest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
import windows_test_baseline as baseline  # noqa: E402


def run_log(directory, tests, done=True):
    """A `flutter test --file-reporter json:...` log. tests: (file, name, result), where result is
    success, failure, error or skipped."""
    events, suites = [], {}
    for index, (file, name, result) in enumerate(tests):
        if file not in suites:
            suites[file] = len(suites)
            windows_path = "D:\\a\\repo\\desktop\\" + file.replace("/", "\\")
            events.append({"type": "suite", "suite": {"id": suites[file], "platform": "vm", "path": windows_path}})
        events.append({"type": "testStart", "test": {"id": index, "name": name, "suiteID": suites[file]}})
        events.append({
            "type": "testDone",
            "testID": index,
            "result": "success" if result == "skipped" else result,
            "skipped": result == "skipped",
            "hidden": name.startswith("loading "),
        })
    if done:
        events.append({"type": "done", "success": False})
    path = Path(directory) / f"run-{len(list(Path(directory).iterdir()))}.json"
    path.write_text("Building flutter tool...\n" + "\n".join(json.dumps(e) for e in events) + "\n", encoding="utf-8")
    return str(path)


class BaselineTest(unittest.TestCase):
    def setUp(self):
        self.dir = tempfile.TemporaryDirectory()
        self.addCleanup(self.dir.cleanup)
        self.baseline = Path(self.dir.name) / "known.txt"
        self.baseline.write_text(baseline.HEADER + "test/posix_test.dart::uses /bin/sh\n", encoding="utf-8")

    def check(self, *logs):
        lines = []
        code = baseline.check(list(logs), str(self.baseline), out=lines.append)
        return code, "\n".join(lines)

    def test_a_recorded_failure_passes_and_a_new_one_fails(self):
        recorded = run_log(self.dir.name, [("test/posix_test.dart", "uses /bin/sh", "failure"), ("test/a_test.dart", "works", "success")])
        self.assertEqual(self.check(recorded)[0], 0)
        new = run_log(self.dir.name, [("test/posix_test.dart", "uses /bin/sh", "failure"), ("test/a_test.dart", "works", "error")])
        code, out = self.check(new)
        self.assertEqual(code, 1)
        self.assertIn("new failure: test/a_test.dart::works", out)

    def test_a_must_pass_file_fails_even_when_recorded(self):
        entry = "test/home_dir_test.dart::homeDir is USERPROFILE on Windows"
        self.baseline.write_text(baseline.HEADER + entry + "\n", encoding="utf-8")
        log = run_log(self.dir.name, [("test/home_dir_test.dart", "homeDir is USERPROFILE on Windows", "failure")])
        code, out = self.check(log)
        self.assertEqual(code, 1)
        self.assertIn("must pass on Windows: " + entry, out)

    def test_a_file_that_does_not_load_is_a_failure(self):
        log = run_log(self.dir.name, [("test/b_test.dart", "loading D:\\a\\repo\\desktop\\test\\b_test.dart", "error")])
        code, out = self.check(log)
        self.assertEqual(code, 1)
        self.assertIn("new failure: test/b_test.dart::(loading)", out)

    def test_a_run_that_did_not_finish_fails(self):
        log = run_log(self.dir.name, [("test/a_test.dart", "works", "success")], done=False)
        code, out = self.check(log)
        self.assertEqual(code, 1)
        self.assertIn("did not finish", out)

    def test_premature_reporter_done_status_fails_check_and_write(self):
        for done in [{"type": "done", "success": None}, {"type": "done"}]:
            with self.subTest(done=done):
                log = run_log(self.dir.name, [("test/posix_test.dart", "uses /bin/sh", "failure")])
                lines = Path(log).read_text(encoding="utf-8").splitlines()
                lines[-1] = json.dumps(done)
                Path(log).write_text("\n".join(lines) + "\n", encoding="utf-8")
                code, out = self.check(log)
                self.assertEqual(code, 1)
                self.assertIn("did not finish", out)
                with contextlib.redirect_stderr(io.StringIO()):
                    self.assertEqual(baseline.write([log], out=lambda _: None), 1)

    def test_started_tests_without_outcomes_fail_even_with_successful_done(self):
        for file in ["test/home_dir_test.dart", "test/shared_test.dart"]:
            with self.subTest(file=file):
                log = run_log(self.dir.name, [(file, "unfinished", "success")])
                lines = [line for line in Path(log).read_text(encoding="utf-8").splitlines()
                         if '"type": "testDone"' not in line]
                lines[-1] = json.dumps({"type": "done", "success": True})
                Path(log).write_text("\n".join(lines) + "\n", encoding="utf-8")
                code, out = self.check(log)
                self.assertEqual(code, 1)
                self.assertIn("did not finish", out)
                with contextlib.redirect_stderr(io.StringIO()):
                    self.assertEqual(baseline.write([log], out=lambda _: None), 1)

    def test_completed_successful_and_recorded_failed_runs_remain_complete(self):
        failed = run_log(self.dir.name, [("test/posix_test.dart", "uses /bin/sh", "failure")])
        self.assertEqual(self.check(failed)[0], 0)
        self.assertEqual(baseline.write([failed], out=lambda _: None), 0)
        passed = run_log(self.dir.name, [("test/shared_test.dart", "finished", "success")])
        lines = Path(passed).read_text(encoding="utf-8").splitlines()
        lines[-1] = json.dumps({"type": "done", "success": True})
        Path(passed).write_text("\n".join(lines) + "\n", encoding="utf-8")
        self.assertEqual(self.check(passed)[0], 0)
        self.assertEqual(baseline.write([passed], out=lambda _: None), 0)

    def test_skips_are_not_failures_and_a_fixed_test_is_named(self):
        log = run_log(self.dir.name, [("test/posix_test.dart", "uses /bin/sh", "success"), ("test/w_test.dart", "on Windows", "skipped")])
        code, out = self.check(log)
        self.assertEqual(code, 0)
        self.assertIn("passes on Windows now", out)

    def test_two_architectures_fail_a_test_that_fails_on_either(self):
        x64 = run_log(self.dir.name, [("test/a_test.dart", "works", "success")])
        arm64 = run_log(self.dir.name, [("test/a_test.dart", "works", "failure")])
        self.assertEqual(self.check(x64, arm64)[0], 1)

    def test_write_records_failures_but_refuses_a_must_pass_one(self):
        lines = []
        log = run_log(self.dir.name, [("test/posix_test.dart", "uses /bin/sh", "failure")])
        self.assertEqual(baseline.write([log], out=lines.append), 0)
        self.assertEqual(lines[-1], "test/posix_test.dart::uses /bin/sh")
        must = run_log(self.dir.name, [("test/viewer_mode_test.dart", "x", "failure")])
        self.assertEqual(baseline.write([must], out=lambda _: None), 1)

    def test_show_lists_one_file_with_skips(self):
        lines = []
        log = run_log(self.dir.name, [("test/w_test.dart", "on Windows a", "skipped"), ("test/w_test.dart", "b", "success")])
        baseline.show([log], "test/w_test.dart", out=lines.append)
        self.assertEqual(lines, ["skipped test/w_test.dart::on Windows a", "success test/w_test.dart::b"])


if __name__ == "__main__":
    unittest.main()