Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

perf(git): show pull requests and CI verdicts before their checks arrive (#636)

Ruben Lütked2606fe50ddf
diff --git a/jarvis/agentic_ide/git_overview.py b/jarvis/agentic_ide/git_overview.py
index da825def46..a76580d0d3 100644
--- a/jarvis/agentic_ide/git_overview.py
+++ b/jarvis/agentic_ide/git_overview.py
@@ -34,6 +34,7 @@ import shutil
import subprocess
import threading
import time
+from collections.abc import Callable
from dataclasses import asdict, dataclass, field
from datetime import datetime
from pathlib import Path
@@ -65,13 +66,34 @@ GITHUB_TTL_S = 60.0
GITHUB_BUSY_TTL_S = 20.0
#: A forced refresh still reuses an answer this young.
GITHUB_MIN_REFRESH_S = 5.0
+#: A read that takes this long says in the log where the time went.
+SLOW_READ_S = 2.0
+
+# GitHub is asked its parts side by side instead of in one query. One query
+# for all of it takes about ten seconds on a repository with many checks per
+# commit — the edge of GitHub's own time limit, where it answers 504. Each
+# part alone is well inside it.
+#
+# The checks behind each pull request's verdict are the slow part by far, so
+# the pull requests are asked twice: once with the verdict alone, which is on
+# screen in about two seconds, and once with the checks, which follow.
+
+#: Every open pull request with the CI verdict of its newest commit, no checks.
+_OPEN_QUICK_QUERY = """
+query($owner: String!, $name: String!) {
+ repository(owner: $owner, name: $name) {
+ url
+ pullRequests(first: 50, states: OPEN, orderBy: {field: UPDATED_AT, direction: DESC}) {
+ nodes {
+ ...PullRequestRow
+ commits(last: 1) { nodes { commit { oid statusCheckRollup { state } } } }
+ }
+ }
+ }
+}
+"""
-# GitHub is asked three things side by side instead of in one query. One
-# query for all of it takes about ten seconds on a repository with many checks
-# per commit — the edge of GitHub's own time limit, where it answers 504. Each
-# part alone is well inside it, and the slowest one sets the wait.
-
-#: Every open pull request with the CI of its newest commit.
+#: The same pull requests with the checks behind each verdict.
_OPEN_QUERY = """
query($owner: String!, $name: String!) {
repository(owner: $owner, name: $name) {
@@ -451,6 +473,10 @@ def parse_github(payload: dict[str, Any], fetched_at: float = 0.0) -> _GitHubSna
commit = (commits[0] or {}).get("commit") if commits else None
commit = commit if isinstance(commit, dict) else {}
entry = node.get("mergeQueueEntry") or {}
+ ci = ci_from_rollup(commit.get("statusCheckRollup"), str(commit.get("oid") or ""))
+ if ci.state != "none" and not ci.url and node.get("url"):
+ # A verdict that came without its checks still has somewhere to go.
+ ci.url = f"{node['url']}/checks"
snap.prs.append(
PullRequest(
number=int(node.get("number") or 0),
@@ -463,7 +489,7 @@ def parse_github(payload: dict[str, Any], fetched_at: float = 0.0) -> _GitHubSna
merged_at=str(node.get("mergedAt") or ""),
updated_at=str(node.get("updatedAt") or ""),
queue_position=entry.get("position") if isinstance(entry, dict) else None,
- ci=ci_from_rollup(commit.get("statusCheckRollup"), str(commit.get("oid") or "")),
+ ci=ci,
conflicting=str(node.get("mergeable") or "").upper() == "CONFLICTING",
review=str(node.get("reviewDecision") or "").lower(),
author=str((node.get("author") or {}).get("login") or ""),
@@ -501,7 +527,15 @@ def _repository(payload: dict[str, Any]) -> dict[str, Any]:
return repo if isinstance(repo, dict) else {}
-def _fetch_github(repo: str, token: str) -> _GitHubSnapshot:
+def _fetch_github(
+ repo: str, token: str, early: Callable[[_GitHubSnapshot], None] | None = None
+) -> _GitHubSnapshot:
+ """GitHub's answer for ``repo``.
+
+ ``early`` is handed a first answer as soon as the quick parts are in: every
+ pull request with its CI verdict and every branch, without the checks
+ behind a verdict and without CI for a branch that has no pull request.
+ """
now = time.time()
owner, _, name = repo.partition("/")
variables = {"owner": owner, "name": name}
@@ -512,19 +546,41 @@ def _fetch_github(repo: str, token: str) -> _GitHubSnapshot:
except github_link.GitHubError as exc: # handed back to the caller below
return exc
+ quick_job = git_local.Job(ask, _OPEN_QUICK_QUERY + _PR_FRAGMENT)
open_job = git_local.Job(ask, _OPEN_QUERY + _PR_FRAGMENT + _ROLLUP_FRAGMENT)
recent_job = git_local.Job(ask, _RECENT_QUERY + _PR_FRAGMENT)
refs_job = git_local.Job(ask, _REFS_CI_QUERY + _ROLLUP_FRAGMENT)
names_job = git_local.Job(_all_refs, token, owner, name)
- opened, recent, refs = open_job.result(), recent_job.result(), refs_job.result()
- names = names_job.result()
- for answer in (opened, refs):
- if isinstance(answer, github_link.GitHubError): # shown in the tab as github.reason/code
- return _GitHubSnapshot(ok=False, reason=str(answer), code=answer.code, fetched_at=now)
+ quick, recent, names = quick_job.result(), recent_job.result(), names_job.result()
if isinstance(recent, github_link.GitHubError):
# Open pull requests and CI landed; only "merged" marks are missing this round.
logger.debug("Git overview: recent pull requests failed: {}", recent.code)
recent = {}
+ if early is not None and not isinstance(quick, github_link.GitHubError):
+ first = _snapshot(quick, recent, {}, names, now)
+ if first.ok:
+ early(first)
+ opened, refs = open_job.result(), refs_job.result()
+ if isinstance(opened, github_link.GitHubError) and not isinstance(
+ quick, github_link.GitHubError
+ ):
+ # The checks behind each verdict are missing this round; the verdicts are not.
+ logger.debug("Git overview: pull request checks failed: {}", opened.code)
+ opened = quick
+ for answer in (opened, refs):
+ if isinstance(answer, github_link.GitHubError): # shown in the tab as github.reason/code
+ return _GitHubSnapshot(ok=False, reason=str(answer), code=answer.code, fetched_at=now)
+ return _snapshot(opened, recent, refs, names, now)
+
+
+def _snapshot(
+ opened: dict[str, Any],
+ recent: dict[str, Any],
+ refs: dict[str, Any],
+ names: list[tuple[str, str, int]],
+ now: float,
+) -> _GitHubSnapshot:
+ """One answer out of the parts GitHub was asked for."""
open_repo, refs_repo = _repository(opened), _repository(refs)
if not open_repo and not refs_repo:
return _GitHubSnapshot(
@@ -549,6 +605,28 @@ def _fetch_github(repo: str, token: str) -> _GitHubSnapshot:
return snap
+def _keep_details(first: _GitHubSnapshot, last: _GitHubSnapshot) -> None:
+ """Fill the quick answer from the last full one wherever nothing moved.
+
+ The quick answer has each pull request's verdict but not the checks behind
+ it, and no CI at all for a branch without a pull request. For a commit the
+ last full answer already covered, what it said is still the best there is.
+ """
+ # A verdict alone cannot tell "running" from "waiting to start".
+ coarse = {"running": "pending"}
+ known = {pr.number: pr.ci for pr in last.prs}
+ for pr in first.prs:
+ old = known.get(pr.number)
+ if old is None or not old.total or old.commit != pr.ci.commit:
+ continue
+ if coarse.get(old.state, old.state) == coarse.get(pr.ci.state, pr.ci.state):
+ pr.ci = old
+ for name, (oid, ci) in list(first.refs.items()):
+ old_ref = last.refs.get(name)
+ if old_ref is not None and old_ref[0] == oid and ci.state == "none":
+ first.refs[name] = old_ref
+
+
def _all_refs(token: str, owner: str, name: str) -> list[tuple[str, str, int]]:
"""Every GitHub branch as (name, tip, commit time), without CI, page by page."""
found: list[tuple[str, str, int]] = []
@@ -614,16 +692,26 @@ class _GitHubCache:
with self._guard:
# Whatever expires it from here on is newer than this read.
self._expired.discard(key)
- fetched = _fetch_github(key, token)
+
+ def early(first: _GitHubSnapshot) -> None:
+ # Pull requests and their verdicts are on screen while the
+ # checks behind them are still on their way.
+ if snap is not None and snap.ok:
+ _keep_details(first, snap)
+ with self._guard:
+ self._entries[key] = first
+
+ fetched = _fetch_github(key, token, early)
with self._guard:
- if fetched.ok or snap is None or not snap.ok:
+ shown = self._entries.get(key)
+ if fetched.ok or shown is None or not shown.ok:
self._entries[key] = fetched
self._failed.pop(key, None)
else:
# GitHub was unreachable for a moment: what was read
# before stays on screen, and the next try waits a turn.
self._failed[key] = time.time()
- fetched = snap
+ fetched = shown
return fetched
finally:
with self._guard:
@@ -860,7 +948,9 @@ def overview(
return RepoOverview(available=False, reason="The workspace folder is missing.")
if shutil.which("git") is None:
return RepoOverview(available=False, reason="Git is not installed.")
+ started = time.perf_counter()
state = git_local.read(path, fresh=refresh)
+ local_done = time.perf_counter()
if state is None:
return RepoOverview(available=False, reason="This folder is not a git repository.")
root = state.root
@@ -882,6 +972,7 @@ def overview(
snap: _GitHubSnapshot | None = None
if github:
snap = _github_for(root, info, refresh, wait_for_github)
+ github_done = time.perf_counter()
# Current and default branch always make the list, however old they are.
keep = {head, info.default_branch}
@@ -920,6 +1011,18 @@ def overview(
# Newest first, like the local list; the later pages arrive in name order.
info.remote_branches.sort(key=lambda row: row.committed_at, reverse=True)
info.pull_requests = _open_pull_requests(snap)
+ total = time.perf_counter() - started
+ if total >= SLOW_READ_S:
+ # Rare, and impossible to explain afterwards without the parts.
+ logger.info(
+ "Git overview: slow read of {} — {:.1f} s (local git {:.1f} s, "
+ "GitHub {:.1f} s, branch rows {:.1f} s)",
+ root,
+ total,
+ local_done - started,
+ github_done - local_done,
+ total - (github_done - started),
+ )
return info
diff --git a/tests/unit/agentic_ide/test_git_control_center.py b/tests/unit/agentic_ide/test_git_control_center.py
index 2e8e0dba49..c8f6e70574 100644
--- a/tests/unit/agentic_ide/test_git_control_center.py
+++ b/tests/unit/agentic_ide/test_git_control_center.py
@@ -155,7 +155,7 @@ def test_unfetched_commits_on_github_are_flagged(
theirs = _commit(other, "theirs.txt")
_git(other, "push", "-q")
- def snapshot(repo_name: str, token: str) -> git_overview._GitHubSnapshot:
+ def snapshot(repo_name: str, token: str, early: object = None) -> git_overview._GitHubSnapshot:
snap = git_overview._GitHubSnapshot(ok=True, fetched_at=git_overview.time.time())
snap.refs["main"] = (theirs[:12], CiStatus())
return snap
@@ -466,7 +466,8 @@ def test_github_is_asked_in_parts_and_a_missing_part_only_costs_its_own_data(
monkeypatch.setattr(github_link, "graphql", fake_graphql)
snap = git_overview._fetch_github("o/r", "t")
- assert sorted(asked) == ["open", "recent", "refs"]
+ # The open pull requests are asked twice: the verdicts first, the checks after.
+ assert sorted(asked) == ["open", "open", "recent", "refs"]
# Recently merged pull requests are missing this round; everything else landed.
assert snap.ok and [pr.number for pr in snap.prs] == [1, 2]
assert (snap.prs[1].conflicting, snap.prs[1].review, snap.prs[1].author) == (
@@ -485,7 +486,9 @@ def test_a_refresh_never_makes_the_tab_wait_for_github(
release = threading.Event()
reads: list[float] = []
- def slow_fetch(repo_name: str, token: str) -> git_overview._GitHubSnapshot:
+ def slow_fetch(
+ repo_name: str, token: str, early: object = None
+ ) -> git_overview._GitHubSnapshot:
reads.append(git_overview.time.time())
if len(reads) > 1:
release.wait(5)
@@ -568,7 +571,7 @@ def test_a_failed_github_read_keeps_the_last_good_answer(monkeypatch: pytest.Mon
]
reads: list[str] = []
- def fetch(repo_name: str, token: str) -> git_overview._GitHubSnapshot:
+ def fetch(repo_name: str, token: str, early: object = None) -> git_overview._GitHubSnapshot:
reads.append(repo_name)
return answers[min(len(reads), len(answers)) - 1]
@@ -644,3 +647,136 @@ def test_an_open_workspace_is_a_folder_threads_may_read(
monkeypatch.setattr(library, "list_projects", lambda include_archived=False: [])
assert thread_folders.allowed_thread_folder(opened) == opened.resolve()
assert thread_folders.allowed_thread_folder(tmp_path) is None
+
+
+# ------------------------------------------------------------------ quick answer
+
+
+def _checks(conclusion: str) -> dict:
+ return {
+ "state": "SUCCESS" if conclusion == "SUCCESS" else "FAILURE",
+ "contexts": {
+ "totalCount": 1,
+ "nodes": [
+ {
+ "__typename": "CheckRun",
+ "name": "CI gate",
+ "status": "COMPLETED",
+ "conclusion": conclusion,
+ "detailsUrl": "https://github.com/o/r/runs/1",
+ }
+ ],
+ },
+ }
+
+
+def _with_ci(node: dict, rollup: dict, oid: str = "a" * 40) -> dict:
+ return {**node, "commits": {"nodes": [{"commit": {"oid": oid, "statusCheckRollup": rollup}}]}}
+
+
+def _github_in_two_steps(monkeypatch: pytest.MonkeyPatch, checks, oid: str = "a" * 40):
+ """A fake GitHub whose checks-bearing answer comes from ``checks()``."""
+
+ def fake_graphql(token: str, query: str, variables: dict | None = None) -> dict:
+ if "pageInfo" in query:
+ return _repo_payload(refs={"pageInfo": {"hasNextPage": False}, "nodes": []})
+ if "states: OPEN" in query and "contexts" not in query:
+ node = _with_ci(_pr_node(1, "a"), {"state": "SUCCESS"}, oid)
+ return _repo_payload(pullRequests={"nodes": [node]})
+ if "states: OPEN" in query:
+ return checks()
+ if "MERGED" in query:
+ return _repo_payload(pullRequests={"nodes": []})
+ return _repo_payload(refs={"nodes": [{"name": "main", "target": {"oid": "0" * 40}}]})
+
+ monkeypatch.setattr(github_link, "graphql", fake_graphql)
+
+
+def _until(done) -> None:
+ deadline = git_overview.time.time() + 5
+ while not done() and git_overview.time.time() < deadline:
+ git_overview.time.sleep(0.02)
+ assert done()
+
+
+def test_pull_requests_and_verdicts_show_before_their_checks_arrive(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ import threading
+
+ release = threading.Event()
+
+ def checks() -> dict:
+ release.wait(5)
+ node = _with_ci(_pr_node(1, "a"), _checks("SUCCESS"))
+ return _repo_payload(pullRequests={"nodes": [node]})
+
+ _github_in_two_steps(monkeypatch, checks)
+ cache = git_overview._GitHubCache()
+ assert cache.get("o/r", "t", wait=False) is None
+ _until(lambda: cache.peek("o/r") is not None)
+ first = cache.peek("o/r")
+ # The verdict is there and leads to the checks page; the checks are still loading.
+ assert first.ok and cache.loading("o/r")
+ assert (first.prs[0].ci.state, first.prs[0].ci.total) == ("success", 0)
+ assert first.prs[0].ci.url == "https://github.com/o/r/pull/1/checks"
+ release.set()
+ _until(lambda: not cache.loading("o/r"))
+ full = cache.peek("o/r")
+ assert (full.prs[0].ci.total, full.prs[0].ci.passed) == (1, 1)
+ assert "main" in full.refs
+
+
+def test_verdicts_stay_when_the_checks_cannot_be_read(monkeypatch: pytest.MonkeyPatch) -> None:
+ def checks() -> dict:
+ raise github_link.GitHubError("GitHub took too long.", code="unreachable")
+
+ _github_in_two_steps(monkeypatch, checks)
+ snap = git_overview._fetch_github("o/r", "t")
+ assert snap.ok and [(pr.number, pr.ci.state) for pr in snap.prs] == [(1, "success")]
+
+
+def test_a_refresh_keeps_the_checks_of_commits_that_did_not_move(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ def snapshot(oid: str, ci: CiStatus) -> git_overview._GitHubSnapshot:
+ snap = git_overview._GitHubSnapshot(ok=True)
+ snap.prs = [
+ git_overview.PullRequest(
+ number=1, title="", url="", state="open", base="main", head="a", ci=ci
+ )
+ ]
+ snap.refs["side"] = (oid, CiStatus())
+ return snap
+
+ detailed = CiStatus(state="success", total=3, passed=3, commit="a" * 12)
+ last = snapshot("b" * 12, detailed)
+ last.refs["side"] = ("b" * 12, CiStatus(state="failure", total=2, failed=1))
+
+ same = snapshot("b" * 12, CiStatus(state="success", commit="a" * 12))
+ git_overview._keep_details(same, last)
+ assert same.prs[0].ci.total == 3 and same.refs["side"][1].state == "failure"
+
+ # A new commit, or a verdict that changed, is not dressed in the old checks.
+ moved = snapshot("c" * 12, CiStatus(state="success", commit="d" * 12))
+ git_overview._keep_details(moved, last)
+ assert moved.prs[0].ci.total == 0 and moved.refs["side"][1].state == "none"
+ turned = snapshot("b" * 12, CiStatus(state="failure", commit="a" * 12))
+ git_overview._keep_details(turned, last)
+ assert turned.prs[0].ci.total == 0
+
+
+def test_a_slow_read_says_where_the_time_went(repo: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ from loguru import logger
+
+ lines: list[str] = []
+ sink = logger.add(lambda message: lines.append(str(message)), level="INFO")
+ try:
+ git_overview.overview(repo, github=False)
+ assert not [line for line in lines if "slow read" in line]
+ monkeypatch.setattr(git_overview, "SLOW_READ_S", 0.0)
+ git_overview.overview(repo, github=False)
+ finally:
+ logger.remove(sink)
+ slow = [line for line in lines if "slow read" in line]
+ assert len(slow) == 1 and "local git" in slow[0] and "GitHub" in slow[0]
diff --git a/tests/unit/agentic_ide/test_git_overview.py b/tests/unit/agentic_ide/test_git_overview.py
index 0db2ec7c35..825c944f56 100644
--- a/tests/unit/agentic_ide/test_git_overview.py
+++ b/tests/unit/agentic_ide/test_git_overview.py
@@ -285,7 +285,9 @@ def test_overview_joins_pull_requests_ci_and_squash_merges(
fresh_head = _git(repo, "rev-parse", "feature/fresh")
calls: list[str] = []
- def fake_fetch(repo_name: str, token: str) -> git_overview._GitHubSnapshot:
+ def fake_fetch(
+ repo_name: str, token: str, early: object = None
+ ) -> git_overview._GitHubSnapshot:
calls.append(repo_name)
snap = parse_github(
_payload(
@@ -360,7 +362,9 @@ def test_a_first_read_lists_local_branches_without_waiting_for_github(
) -> None:
release = threading.Event()
- def slow_fetch(repo_name: str, token: str) -> git_overview._GitHubSnapshot:
+ def slow_fetch(
+ repo_name: str, token: str, early: object = None
+ ) -> git_overview._GitHubSnapshot:
release.wait(5)
return parse_github(_payload([]), fetched_at=git_overview.time.time())