Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

22.3 KB
Raw Blame
"""What the Overview page reads: machine facts from collectors (never trusted), sparklines, the all-clear summary
and a server's event timeline."""
import json
import os
from datetime import datetime, timedelta, timezone

os.environ.setdefault("DATABASE_URL", "sqlite://")
os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")

import pytest
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool

from src.collector import collector, system_helper
from src.backend import dashboard, main
from src.backend.models import Base, ContainerMetric, HostMetric, IncidentAlert, PendingCollector, Server, SystemEvent, User
from tests.fakes import FakeRedis

TOKEN = "token-one"
NOW = datetime(2026, 10, 13, 12, 0, tzinfo=timezone.utc)


@pytest.fixture
def app(monkeypatch):
    engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
    Base.metadata.create_all(engine)
    Session = sessionmaker(bind=engine)
    with Session() as seed:
        seed.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin", is_active=True))
        seed.add(Server(id=1, owner_id=1, hostname="alpha", check_interval_seconds=60, collector_token_hash=main._hash_secret(TOKEN), machine_id="machine-1", created_at=NOW - timedelta(days=60)))
        seed.commit()

    def get_db():
        session = Session()
        try:
            yield session
        finally:
            session.close()

    main.app.dependency_overrides[main.get_db] = get_db
    monkeypatch.setattr(main, "redis_client", FakeRedis())
    monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1, "role": "admin"})
    monkeypatch.setattr(main, "_require_admin", lambda session_id, db: {"id": 1, "role": "admin"})
    monkeypatch.setattr(main, "_is_admin", lambda db, user_id: True)
    client = TestClient(main.app, raise_server_exceptions=False)
    client.Session = Session
    yield client
    main.app.dependency_overrides.clear()


def ingest(client, body):
    return client.post("/monitoring/metrics/ingest", content=json.dumps({"server_id": 1, **body}), headers={"Authorization": f"Bearer {TOKEN}", "Content-Type": "application/json"})


def overview_server(client):
    return client.get("/monitoring/overview").json()["servers"][0]


# --- machine facts --------------------------------------------------------------------------

def test_system_info_keeps_known_fields_and_drops_the_rest(app):
    system = {
        "kernel": "6.8.0-84-generic", "arch": "x86_64", "cpu_model": "Intel Core i5-8500T", "swap_total_gb": 4.0, "updates_pending": 23, "updates_security": 4,
        "auto_updates": True, "helper_installed": False, "reboot_kernel": "6.8.0-85-generic", "clock_offset_ms": 0.4, "secret": "x", "html": "<script>",
        "temperatures": [{"label": "coretemp", "celsius": 52.0}, {"label": "bad", "celsius": 9999}, "junk"],
        "interfaces": [{"name": "eth0", "network": "10.0.0.12/24", "rx_mbps": 1.8, "tx_mbps": float("1e99"), "speed_mbit": 1000, "extra": 1}, {"name": "", "network": "x"}, {"name": "wg0", "network": "not a network"}],
        "mount_devices": {"/": "nvme0n1p2", "/x": 5},
        "firewall_allowed": [{"proto": "tcp", "port": 22, "file": "nftables"}, {"proto": "icmp", "port": 1, "file": "nftables"}, {"proto": "tcp", "port": 80, "file": "/etc/passwd"}],
    }
    assert ingest(app, {"server": {"system": system}, "host": {"cpu_utilization_pct": 5}}).status_code == 200

    stored = overview_server(app)["system"]
    assert stored["kernel"] == "6.8.0-84-generic" and stored["updates_security"] == 4 and stored["auto_updates"] is True and stored["reboot_kernel"] == "6.8.0-85-generic"
    assert "secret" not in stored and "html" not in stored
    assert stored["temperatures"] == [{"label": "coretemp", "celsius": 52.0}]
    assert stored["interfaces"] == [{"name": "eth0", "network": "10.0.0.0/24", "rx_mbps": 1.8, "speed_mbit": 1000}, {"name": "wg0"}]
    assert stored["mount_devices"] == {"/": "nvme0n1p2"}
    assert stored["firewall_allowed"] == [{"proto": "tcp", "port": 22, "file": "nftables"}]


@pytest.mark.parametrize("junk", [None, 5, "text", [], {"kernel": 5, "interfaces": "x", "temperatures": {}, "mount_devices": [], "firewall_allowed": 3, "updates_pending": -1, "auto_updates": "yes"}])
def test_malformed_system_info_never_breaks_a_check_in(app, junk):
    assert ingest(app, {"server": {"system": junk}, "host": {"cpu_utilization_pct": 5}}).status_code == 200
    assert isinstance(overview_server(app)["system"], dict)


def test_containers_carry_their_project_ports_and_volumes(app):
    container = {"container_id": "c" * 64, "name": "gitlab", "status": "running", "project": "gitlab", "ports": "8080->80/tcp", "volumes": "/srv/gitlab/config, /srv/gitlab/data", "started_at": "2026-10-03T09:12:00Z"}
    ingest(app, {"host": {"cpu_utilization_pct": 5}, "containers": [container, {"container_id": "d" * 64, "name": "solo", "status": "running", "project": 7}]})

    by_name = {item["name"]: item for item in overview_server(app)["containers"]}
    assert (by_name["gitlab"]["project"], by_name["gitlab"]["ports"], by_name["gitlab"]["started_at"]) == ("gitlab", "8080->80/tcp", "2026-10-03T09:12:00Z")
    assert by_name["solo"]["project"] is None
    assert app.get(f"/monitoring/servers/1/containers/{'c' * 64}").json()["volumes"] == "/srv/gitlab/config, /srv/gitlab/data"


# --- adding a server ------------------------------------------------------------------------

def test_a_pending_collector_shows_what_it_found_and_can_be_named(app):
    registered = app.post("/monitoring/collectors/register", json={"hostname": "box", "machine_id": "machine-9", "details": {"cpu_cores": 4, "containers": 35, "mounts": 2, "helper_installed": False, "dashboard_host": True, "junk": "x"}})
    assert registered.status_code == 200
    [pending] = app.get("/monitoring/overview").json()["pending_collectors"]
    assert pending["details"] == {"cpu_cores": 4, "containers": 35, "mounts": 2, "helper_installed": False, "dashboard_host": True}

    assert app.post(f"/monitoring/pending-collectors/{pending['id']}/approve", json={"hostname": "alpha"}).status_code == 409  # taken
    assert app.post(f"/monitoring/pending-collectors/{pending['id']}/approve", json={"hostname": "media-box"}).status_code == 200
    with app.Session() as db:
        server = db.query(Server).filter_by(machine_id="machine-9").one()
        assert (server.hostname, server.name_locked) == ("media-box", True)
        token = "token-nine"
        server.collector_token_hash = main._hash_secret(token)
        db.commit()
        server_id = server.id
    # The machine keeps calling itself "box"; the chosen name stays.
    app.post("/monitoring/metrics/ingest", content=json.dumps({"server_id": server_id, "server": {"hostname": "box"}, "host": {"cpu_utilization_pct": 1}}), headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"})
    with app.Session() as db:
        assert db.get(Server, server_id).hostname == "media-box"


def test_keeping_the_machines_own_name_does_not_lock_it(app):
    app.post("/monitoring/collectors/register", json={"hostname": "box", "machine_id": "machine-9"})
    [pending] = app.get("/monitoring/overview").json()["pending_collectors"]
    app.post(f"/monitoring/pending-collectors/{pending['id']}/approve", json={"hostname": "box"})
    with app.Session() as db:
        assert db.query(Server).filter_by(machine_id="machine-9").one().name_locked is False


# --- sparklines -----------------------------------------------------------------------------

def test_downsample_averages_and_keeps_gaps():
    assert dashboard.downsample([1, 2, 3], limit=5) == [1, 2, 3]
    assert dashboard.downsample([0, 10, None, None, 4, 6], limit=3) == [5, None, 5]


def test_sparklines_cover_recent_samples_only(app):
    with app.Session() as db:
        now = datetime.now(timezone.utc)
        for minutes_ago, cpu in ((90, 99.0), (10, 20.0), (5, 40.0)):
            db.add(HostMetric(server_id=1, time=now - timedelta(minutes=minutes_ago), cpu_utilization_pct=cpu))
            db.add(ContainerMetric(server_id=1, container_id="c1", name="web", status="running", cpu_pct=cpu / 2, time=now - timedelta(minutes=minutes_ago)))
        db.commit()

    assert app.get("/monitoring/sparklines?minutes=30").json()["servers"] == {"1": [20.0, 40.0]}
    assert app.get("/monitoring/servers/1/containers/sparklines?minutes=15").json()["containers"] == {"c1": [10.0, 20.0]}
    assert app.get("/monitoring/servers/99/containers/sparklines").status_code == 404
    live = app.get("/monitoring/servers/1/live?minutes=30").json()["samples"]
    assert [sample["cpu"] for sample in live] == [20.0, 40.0] and set(live[0]) == {"time", "cpu", "ram", "load", "swap", "net_sent_mb", "net_recv_mb"}


# --- the all-clear summary ------------------------------------------------------------------

def test_longest_quiet_stretch_is_the_biggest_gap_between_incidents():
    start, end = NOW - timedelta(days=10), NOW
    spans = [(start + timedelta(days=1), start + timedelta(days=2)), (start + timedelta(days=1, hours=12), start + timedelta(days=3)), (start + timedelta(days=9), end)]
    assert dashboard.longest_quiet_seconds(spans, start, end) == timedelta(days=6).total_seconds()
    assert dashboard.longest_quiet_seconds([], start, end) == timedelta(days=10).total_seconds()
    assert dashboard.longest_quiet_seconds([(start - timedelta(days=1), end + timedelta(days=1))], start, end) == 0


def test_quiet_summary_names_the_last_incident_and_ignores_notices(app):
    with app.Session() as db:
        db.add(IncidentAlert(server_id=1, severity="CRITICAL", title="gitlab is in a restart loop", description="d", status="resolved", rule_key="container:gitlab:restart_loop", created_at=NOW - timedelta(hours=3), resolved_at=NOW - timedelta(hours=2)))
        db.add(IncidentAlert(server_id=1, severity="INFO", title="alpha needs a reboot", description="d", status="resolved", rule_key="system:reboot_required", created_at=NOW - timedelta(hours=1), resolved_at=NOW - timedelta(minutes=5)))
        db.commit()
        summary = dashboard.quiet_summary(db, NOW)

    assert summary["last_incident"]["title"] == "gitlab is in a restart loop" and summary["last_incident"]["hostname"] == "alpha"
    assert summary["quiet_since"] == (NOW - timedelta(hours=2)).isoformat()
    assert summary["longest_quiet_seconds_this_month"] == (NOW - timedelta(hours=3) - NOW.replace(day=1, hour=0)).total_seconds()


def test_overview_reports_quiet_only_when_nothing_needs_attention(app):
    assert app.get("/monitoring/overview").json()["quiet"] is not None
    with app.Session() as db:
        db.add(IncidentAlert(server_id=1, severity="WARNING", title="t", description="d", status="active", rule_key="host:memory"))
        db.add(IncidentAlert(server_id=1, severity="INFO", title="n", description="d", status="active", rule_key="system:reboot_required"))
        db.commit()
    body = app.get("/monitoring/overview").json()
    assert body["quiet"] is None and body["alerts_by_severity"] == {"WARNING": 1, "INFO": 1} and body["servers"][0]["alerts_by_severity"] == {"WARNING": 1, "INFO": 1}
    assert {alert["rule_key"] for alert in body["alerts"]} == {"host:memory", "system:reboot_required"}


# --- a server's timeline --------------------------------------------------------------------

def test_events_merge_recorded_events_with_alerts_newest_first(app):
    with app.Session() as db:
        db.add(SystemEvent(server_id=1, time=NOW - timedelta(hours=5), kind="port_opened", detail="tcp/8081 (all interfaces) started listening"))
        db.add(SystemEvent(server_id=1, time=NOW - timedelta(hours=9), kind="container_added", detail="Container web appeared"))
        db.add(SystemEvent(server_id=1, time=NOW - timedelta(days=20), kind="reboot", detail="old"))
        db.add(IncidentAlert(server_id=1, severity="WARNING", title="Memory usage on alpha is 93%", description="d", status="resolved", rule_key="host:memory", created_at=NOW - timedelta(hours=2), resolved_at=NOW - timedelta(hours=1, minutes=42)))
        db.add(IncidentAlert(server_id=1, severity="CRITICAL", title="Server alpha is offline", description="d", status="resolved", rule_key=main.SERVER_OFFLINE_RULE, created_at=NOW - timedelta(days=3), resolved_at=NOW - timedelta(days=3) + timedelta(hours=5)))
        db.commit()
        events = dashboard.server_events(db, db.get(Server, 1), 7, NOW)

    assert [(event["source"], event["kind"]) for event in events] == [
        ("alert", "alert_resolved"), ("alert", "alert_opened"), ("network", "port_opened"), ("container", "container_added"), ("collector", "collector_resumed"), ("collector", "collector_stopped"),
    ]
    assert events[0]["duration_seconds"] == 18 * 60 and events[1]["resolved"] is True
    assert app.get("/monitoring/servers/1/events?days=7").status_code == 200 and app.get("/monitoring/servers/99/events").status_code == 404


# --- what the collector reads ---------------------------------------------------------------

def test_firewall_allowed_names_only_ports_that_are_listening(tmp_path, monkeypatch):
    (tmp_path / "etc/ufw").mkdir(parents=True)
    (tmp_path / "etc/ufw/user.rules").write_text("-A ufw-user-input -p tcp --dport 22 -j ACCEPT\n-A ufw-user-input -p tcp -m multiport --dports 80,443 -j ACCEPT\n-A ufw-user-input -p tcp --dport 5432 -j DROP\n")
    (tmp_path / "etc/nftables.conf").write_text("table inet filter {\n chain input {\n  udp dport { 1194, 51820 } accept\n  tcp dport 9000-9002 accept\n }\n}\n")
    monkeypatch.setattr(collector, "HOST_ROOT", tmp_path)
    listeners = [{"proto": "tcp", "port": 22, "scope": "any"}, {"proto": "tcp", "port": 443, "scope": "any"}, {"proto": "tcp", "port": 5432, "scope": "any"}, {"proto": "udp", "port": 1194, "scope": "any"}, {"proto": "tcp", "port": 9001, "scope": "any"}]

    assert collector.firewall_allowed(listeners) == [
        {"proto": "tcp", "port": 22, "file": "ufw"}, {"proto": "tcp", "port": 443, "file": "ufw"}, {"proto": "tcp", "port": 9001, "file": "nftables"}, {"proto": "udp", "port": 1194, "file": "nftables"},
    ]
    assert collector.firewall_allowed(None) == []


def test_updates_come_from_the_files_apt_leaves_behind(tmp_path, monkeypatch):
    for directory in ("etc/apt/apt.conf.d", "var/lib/update-notifier", "boot"):
        (tmp_path / directory).mkdir(parents=True)
    (tmp_path / "etc/debian_version").write_text("12\n")
    (tmp_path / "var/lib/update-notifier/updates-available").write_text("\n23 updates can be applied immediately.\n4 of these updates are standard security updates.\n")
    (tmp_path / "etc/apt/apt.conf.d/20auto-upgrades").write_text('APT::Periodic::Update-Package-Lists "1";\nAPT::Periodic::Unattended-Upgrade "1";\n')
    for name in ("vmlinuz-6.8.0-9-generic", "vmlinuz-6.8.0-84-generic", "vmlinuz-6.8.0-85-generic", "vmlinuz", "initrd.img-6.8.0-99-generic"):
        (tmp_path / "boot" / name).write_text("")
    monkeypatch.setattr(collector, "HOST_ROOT", tmp_path)

    info = collector.collect_updates("6.8.0-84-generic", True)
    assert (info["updates_pending"], info["updates_security"], info["auto_updates"], info["reboot_kernel"]) == (23, 4, True, "6.8.0-85-generic")
    assert "reboot_kernel" not in collector.collect_updates("6.8.0-85-generic", True)
    assert "reboot_kernel" not in collector.collect_updates("6.8.0-84-generic", False)


def test_interfaces_report_their_network_and_traffic_since_the_last_look(tmp_path, monkeypatch):
    net = tmp_path / "1/net"
    net.mkdir(parents=True)
    header = "Inter-|   Receive |  Transmit\n face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n"
    row = lambda name, rx, tx: f"{name}: {rx} 1 0 0 0 0 0 0 {tx} 1 0 0 0 0 0 0\n"  # noqa: E731
    (net / "route").write_text("Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\n" "eth0\t00000000\t0100000A\t0003\t0\t0\t0\t00000000\n" "eth0\t0000000A\t00000000\t0001\t0\t0\t0\t00FFFFFF\n")
    monkeypatch.setattr(collector, "HOST_PROC", tmp_path)
    monkeypatch.setattr(collector, "HOST_ROOT", tmp_path)
    collector._interface_previous.clear()

    (net / "dev").write_text(header + row("lo", 5, 5) + row("eth0", 1000, 2000) + row("veth12ab", 9, 9) + row("idle0", 0, 0))
    assert collector.collect_interfaces(now=100.0) == [{"name": "eth0", "network": "10.0.0.0/24"}]
    (net / "dev").write_text(header + row("eth0", 1000 + 4 * collector.MiB, 2000 + 2 * collector.MiB))
    assert collector.collect_interfaces(now=102.0) == [{"name": "eth0", "network": "10.0.0.0/24", "rx_mbps": 2.0, "tx_mbps": 1.0}]
    collector._interface_previous.clear()


# --- data that needs more than the collector has ----------------------------------------------

def _fake_host_proc(tmp_path):
    """A host /proc with sshd listening on tcp/22, a resolver on udp/53 and a process that names itself oddly."""
    net = tmp_path / "1/net"
    net.mkdir(parents=True)
    header = "  sl  local_address rem_address   st tx_queue rx_queue tr tm->when retrnsmt   uid  timeout inode\n"
    (net / "tcp").write_text(header + "   0: 00000000:0016 00000000:0000 0A 00000000:00000000 00:00000000 00000000     0        0 1001 1 x\n"
                             "   1: 0100007F:1F90 00000000:0000 0A 00000000:00000000 00:00000000 00000000  1000        0 1003 1 x\n"
                             "   2: 0A00000C:D431 0A000015:01BB 01 00000000:00000000 00:00000000 00000000  1000        0 1009 1 x\n")
    (net / "udp").write_text(header + "  10: 3500007F:0035 00000000:0000 07 00000000:00000000 00:00000000 00000000   101        0 1002 2 x 0\n")
    for pid, name, inodes in ((1, "systemd", [5]), (812, "sshd", [1001, 7]), (640, "systemd-resolve", [1002]), (3000, "evil\x1b[31m\nname", [1003]), (3001, "client", [1009])):
        fd = tmp_path / str(pid) / "fd"
        fd.mkdir(parents=True, exist_ok=True)
        (tmp_path / str(pid) / "comm").write_text(name + "\n")
        for number, inode in enumerate(inodes):
            os.symlink(f"socket:[{inode}]", fd / str(number))
        os.symlink("/etc/shadow", fd / "99")
    return tmp_path


def test_port_owners_name_the_process_behind_each_listening_port(tmp_path):
    proc = _fake_host_proc(tmp_path)

    assert system_helper.listening_sockets(proc) == {1001: ("tcp", 22), 1003: ("tcp", 8080), 1002: ("udp", 53)}   # the connected socket is not a listener
    owners = system_helper.port_owners(proc)
    assert owners == [{"proto": "tcp", "port": 22, "process": "sshd"}, {"proto": "udp", "port": 53, "process": "systemd-resolve"}, {"proto": "tcp", "port": 8080, "process": "evil?[31m?name"}]
    assert system_helper.port_owners(tmp_path / "missing") == []


def test_the_collector_only_trusts_a_fresh_well_formed_port_owners_file(tmp_path):
    path = tmp_path / "ports.json"
    now = 1_000_000.0
    stamp = datetime.fromtimestamp(now, timezone.utc).isoformat(timespec="seconds")
    assert collector.collect_port_owners(path, now) is None   # helper not running
    path.write_text(json.dumps({"generated_at": stamp, "listeners": [{"proto": "tcp", "port": 22, "process": "sshd"}, {"proto": "icmp", "port": 1, "process": "x"}, {"proto": "tcp", "port": "22", "process": "x"}, "junk"]}))
    assert collector.collect_port_owners(path, now) == [{"proto": "tcp", "port": 22, "process": "sshd"}]
    assert collector.collect_port_owners(path, now + 3600) is None   # stale
    path.write_text("{not json")
    assert collector.collect_port_owners(path, now) is None
    link = tmp_path / "link.json"
    os.symlink(path, link)
    assert collector.collect_port_owners(link, now) is None   # never follows a symlink


def test_docker_disk_usage_and_port_owners_are_validated_before_they_are_stored(app):
    system = {
        "docker_disk": {"images_gb": 41.2, "images_unused_gb": 38.0, "build_cache_gb": 2.5, "build_cache_unused_gb": float("inf"), "images": 60, "images_unused": 44, "names": ["secret"]},
        "disk_io_errors": 3,
        "listener_owners": [{"proto": "tcp", "port": 22, "process": "sshd"}, {"proto": "tcp", "port": 99999, "process": "x"}, {"proto": "tcp", "port": 80, "process": "a" * 200}, {"proto": "tcp", "port": 81, "process": ""}],
    }
    ingest(app, {"server": {"system": system}, "host": {"cpu_utilization_pct": 5}})
    stored = overview_server(app)["system"]

    assert stored["docker_disk"] == {"images_gb": 41.2, "images_unused_gb": 38.0, "build_cache_gb": 2.5, "images": 60, "images_unused": 44}
    assert stored["disk_io_errors"] == 3
    assert stored["listener_owners"] == [{"proto": "tcp", "port": 22, "process": "sshd"}, {"proto": "tcp", "port": 80, "process": "a" * 64}]
    # A collector without the helper sends no list at all, which is different from an empty one.
    ingest(app, {"server": {"system": {"kernel": "6.8"}}, "host": {"cpu_utilization_pct": 5}})
    assert "listener_owners" not in overview_server(app)["system"]


def test_disk_error_counter_sums_the_disks_that_have_one(tmp_path, monkeypatch):
    for disk, value in (("sda", "0x3"), ("sdb", "0x0a"), ("nvme0n1", None)):
        (tmp_path / "sys/block" / disk / "device").mkdir(parents=True)
        if value:
            (tmp_path / "sys/block" / disk / "device/ioerr_cnt").write_text(value + "\n")
    monkeypatch.setattr(collector, "HOST_ROOT", tmp_path)
    assert collector.disk_io_errors() == 13
    (tmp_path / "sys/block/sda/device/ioerr_cnt").unlink()
    (tmp_path / "sys/block/sdb/device/ioerr_cnt").unlink()
    assert collector.disk_io_errors() is None


def test_docker_disk_usage_is_asked_for_rarely_and_survives_an_old_proxy(monkeypatch):
    calls = []
    answer = {"images_bytes": 2 * collector.GiB, "images_unused_bytes": collector.GiB, "build_cache_bytes": 0, "build_cache_unused_bytes": 0, "images": 5, "images_unused": 2}
    monkeypatch.setattr(collector, "_docker_request", lambda path, endpoint: calls.append(endpoint) or answer)
    monkeypatch.setattr(collector, "_docker_disk", {"value": None, "next": 0.0})

    collector._remember_docker_disk("sock", now=10.0)
    collector._remember_docker_disk("sock", now=20.0)
    assert calls == ["/system/df"] and collector._docker_disk["value"] == {"images_gb": 2.0, "images_unused_gb": 1.0, "build_cache_gb": 0.0, "build_cache_unused_gb": 0.0, "images": 5, "images_unused": 2}

    def refused(path, endpoint):
        raise ValueError("403")
    monkeypatch.setattr(collector, "_docker_request", refused)
    collector._remember_docker_disk("sock", now=10.0 + collector.DOCKER_DISK_EVERY_SECONDS + 1)
    assert collector._docker_disk["value"]["images_gb"] == 2.0   # the last known figure stays