Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
feat(appshot): bendable arrows and points on every fresh marking in the area picker
rubenluetke10-beep387cde34a162
diff --git a/jarvis/appshot/picker/annotate.py b/jarvis/appshot/picker/annotate.pyindex c357114651..5357768ac2 100644--- a/jarvis/appshot/picker/annotate.py+++ b/jarvis/appshot/picker/annotate.py@@ -244,8 +244,58 @@ def qt_measure(text: str, size: float) -> float:# ---------------------------------------------------------------------------def tapered_arrow_outline(start: mm.Point, end: mm.Point, width: float) -> list[mm.Point]:+def _curved_outline(start: mm.Point, end: mm.Point, width: float, bend: mm.Point) -> list[mm.Point]:+ """The tapered arrow along a curve: the same profile, following the bend."""+ path = mm.segment_path(mm.Shape(mm.ARROW, points=[start, end], bend=bend), steps=48)+ lengths = [0.0]+ for a, b in zip(path, path[1:], strict=False):+ lengths.append(lengths[-1] + math.hypot(b[0] - a[0], b[1] - a[1]))+ length = lengths[-1]+ if length < 2:+ return []+ head = min(max(12.0, width * 4.6 + 6.0), length * 0.62)+ barb = head * 0.56+ neck_at = length - head * 0.7+ tail = max(0.6, width * 0.14)+ neck = max(1.4, width * 0.78)++ def normal(i: int) -> tuple[float, float]:+ a = path[max(0, i - 1)]+ b = path[min(len(path) - 1, i + 1)]+ dx, dy = b[0] - a[0], b[1] - a[1]+ span = math.hypot(dx, dy) or 1.0+ return (-dy / span, dx / span)++ left: list[mm.Point] = []+ right: list[mm.Point] = []+ for i, (point, along) in enumerate(zip(path, lengths, strict=False)):+ if along > neck_at:+ break+ half = tail + (neck - tail) * (along / max(1.0, neck_at))+ nx, ny = normal(i)+ left.append((point[0] + nx * half, point[1] + ny * half))+ right.append((point[0] - nx * half, point[1] - ny * half))+ # The head points along the curve's last stretch.+ tx, ty = end[0] - bend[0], end[1] - bend[1]+ span = math.hypot(tx, ty) or 1.0+ ux, uy = tx / span, ty / span+ nx, ny = -uy, ux+ base = (end[0] - ux * head, end[1] - uy * head)+ return [+ *left,+ (base[0] + nx * barb, base[1] + ny * barb),+ end,+ (base[0] - nx * barb, base[1] - ny * barb),+ *reversed(right),+ ]+++def tapered_arrow_outline(+ start: mm.Point, end: mm.Point, width: float, bend: mm.Point | None = None+) -> list[mm.Point]:"""The tapered arrow, tail to tip and back (``taperedArrowOutline`` in TS)."""+ if bend is not None:+ return _curved_outline(start, end, width, bend)dx, dy = end[0] - start[0], end[1] - start[1]length = math.hypot(dx, dy)if length < 2:@@ -293,7 +343,7 @@ def _soft_shadow(painter: QPainter, path: QPainterPath, blur: float, dy: float,def _tapered_arrow(painter: QPainter, shape: mm.Shape, colour: QColor) -> None:- outline = tapered_arrow_outline(shape.points[0], shape.points[-1], shape.width)+ outline = tapered_arrow_outline(shape.points[0], shape.points[-1], shape.width, shape.bend)if not outline:returnpath = QPainterPath()@@ -307,8 +357,46 @@ def _tapered_arrow(painter: QPainter, shape: mm.Shape, colour: QColor) -> None:painter.drawPath(path)+def _classic_head(painter: QPainter, tip: mm.Point, toward: mm.Point, shape: mm.Shape) -> None:+ """A filled head at ``tip``, pointing away from ``toward``."""+ colour = _qcolor(shape.color)+ angle = math.atan2(tip[1] - toward[1], tip[0] - toward[0])+ head = max(10.0, shape.width * 4.0)+ painter.setPen(Qt.PenStyle.NoPen)+ painter.setBrush(colour)+ painter.drawPolygon(+ _polygon(+ [+ tip,+ (tip[0] - math.cos(angle - 0.45) * head, tip[1] - math.sin(angle - 0.45) * head),+ (tip[0] - math.cos(angle + 0.45) * head, tip[1] - math.sin(angle + 0.45) * head),+ ]+ )+ )+++def _segment_qpath(shape: mm.Shape) -> QPainterPath:+ start, end = shape.points[0], shape.points[-1]+ path = QPainterPath(QPointF(*start))+ if shape.bend is None:+ path.lineTo(QPointF(*end))+ else:+ path.quadTo(QPointF(*shape.bend), QPointF(*end))+ return path++def _classic_arrow(painter: QPainter, start: mm.Point, end: mm.Point, shape: mm.Shape) -> None:colour = _qcolor(shape.color)+ if shape.bend is not None:+ pen = QPen(colour, shape.width)+ pen.setCapStyle(Qt.PenCapStyle.RoundCap)+ painter.setPen(pen)+ painter.setBrush(Qt.BrushStyle.NoBrush)+ painter.drawPath(_segment_qpath(shape))+ _classic_head(painter, end, shape.bend, shape)+ if shape.style == "double":+ _classic_head(painter, start, shape.bend, shape)+ returnangle = math.atan2(end[1] - start[1], end[0] - start[0])head = max(10.0, shape.width * 4.0)back = (end[0] - math.cos(angle) * head * 0.8, end[1] - math.sin(angle) * head * 0.8)@@ -393,10 +481,11 @@ def paint_shape(pen = QPen(colour, shape.width)pen.setCapStyle(Qt.PenCapStyle.RoundCap)painter.setPen(pen)- painter.drawLine(QPointF(*shape.points[0]), QPointF(*shape.points[-1]))+ painter.setBrush(Qt.BrushStyle.NoBrush)+ painter.drawPath(_segment_qpath(shape))elif kind == mm.ARROW:start, end = shape.points[0], shape.points[-1]- if shape.style == "classic":+ if shape.style == "classic" or (shape.style == "double" and shape.bend is not None):_classic_arrow(painter, start, end, shape)elif shape.style == "double":_classic_arrow(painter, start, end, shape)@@ -539,10 +628,11 @@ def paint_selection_frame(painter: QPainter, sel: QRectF) -> None:def paint_picked(painter: QPainter, shape: mm.Shape) -> None:- """The selected marking: a dashed frame (not for lines) and its grips."""+ """The selected marking's points, plus a dashed frame where the points alone+ would float (text, strokes)."""painter.save()painter.setRenderHint(QPainter.RenderHint.Antialiasing)- if shape.kind not in mm.SEGMENT_KINDS:+ if shape.kind == mm.TEXT or shape.kind in mm.STROKE_KINDS:box = _qrect(mm.bounds(shape, qt_measure)).adjusted(-6, -6, 6, 6)painter.setBrush(Qt.BrushStyle.NoBrush)dark = QPen(QColor(0, 0, 0, 150), 1.5, Qt.PenStyle.DashLine)@@ -1147,8 +1237,16 @@ class Toolbar(QFrame):button.setToolTip(self._labels.get(kind, value))button.clicked.connect(lambda _c=False, g=group, v=value: self.option_chosen.emit(g, v))self._options.addWidget(button)+ # Shown now, not on the layout's queued show: a hidden button does+ # not count when the bar measures itself, and the new looks then+ # pile up on top of each other and the colours.+ button.show()self._option_buttons.append(button)self._optsep.setVisible(bool(self._option_buttons))+ layout = self.layout()+ if layout is not None:+ layout.invalidate()+ layout.activate()self.adjustSize()diff --git a/jarvis/appshot/picker/markup_model.py b/jarvis/appshot/picker/markup_model.pyindex 592f4d445c..24d23c4e62 100644--- a/jarvis/appshot/picker/markup_model.py+++ b/jarvis/appshot/picker/markup_model.py@@ -110,6 +110,8 @@ HANDLE_GRAB_PX = 8.0MIN_SELECTION_PX = 8.0#: The smallest text and counter a grip can shrink them to (px).MIN_TEXT_PX = 8.0+#: A bend grip dropped this close to the straight line straightens it again.+STRAIGHT_SNAP_PX = 4.0@dataclass(slots=True)@@ -119,7 +121,8 @@ class Shape:box kinds and segments: ``[start, end]``; strokes: the whole path; text:``[top-left]``; counter: ``[centre]``. ``size`` is the text height or thecounter's radius; ``style`` the arrow or text look; ``mode`` pixelate or- blur for a redaction.+ blur for a redaction. ``bend`` is the control point of a curved arrow or+ line (a quadratic curve from start to end); ``None`` keeps it straight."""kind: str@@ -131,6 +134,7 @@ class Shape:size: float = 0.0style: str = ""mode: str = ""+ bend: Point | None = Nonedef copy(self) -> Shape:return replace(self, points=list(self.points))@@ -158,15 +162,20 @@ def counter_size(width: float) -> float:def grab_scope(tool: str) -> str:"""What a press with ``tool`` may take hold of instead of drawing.- ``any``: the select tool picks up every marking; ``selected``: a shape- tool reshapes or moves only the marking it has selected (the one it just- drew); ``none``: pen, highlighter and counter always draw.+ ``any``: the select tool picks up every marking, areas included;+ ``shapes``: every other drawing tool picks up any marking under the+ pointer (not areas — they cover what is drawn in them), so anything+ drawn can be moved and reshaped again at once; ``grips``: pen and+ highlighter only take the selected marking's points, so writing over+ earlier ink never drags it along; ``none``: the background switch."""if tool == MOVE:return "any"- if tool in (PEN, HIGHLIGHT, COUNTER, BACKGROUND):+ if tool in (PEN, HIGHLIGHT):+ return "grips"+ if tool == BACKGROUND:return "none"- return "selected"+ return "shapes"# --------------------------------------------------------------------------@@ -197,6 +206,31 @@ def constrain(kind: str, start: Point, end: Point) -> Point:return end+def curve_at(start: Point, bend: Point, end: Point, t: float) -> Point:+ """The point at ``t`` on the quadratic curve start - bend - end."""+ u = 1.0 - t+ return (+ u * u * start[0] + 2 * u * t * bend[0] + t * t * end[0],+ u * u * start[1] + 2 * u * t * bend[1] + t * t * end[1],+ )+++def segment_path(shape: Shape, steps: int = 32) -> list[Point]:+ """An arrow's or line's path as points: the two ends, or the sampled curve."""+ start, end = shape.points[0], shape.points[-1]+ if shape.bend is None:+ return [start, end]+ return [curve_at(start, shape.bend, end, i / steps) for i in range(steps + 1)]+++def segment_middle(shape: Shape) -> Point:+ """Where the bend grip sits: the middle of the line or of the curve."""+ start, end = shape.points[0], shape.points[-1]+ if shape.bend is None:+ return ((start[0] + end[0]) / 2, (start[1] + end[1]) / 2)+ return curve_at(start, shape.bend, end, 0.5)++def label_pad(shape: Shape) -> float:return shape.size * 0.4 if shape.kind == TEXT and shape.style == "label" else 0.0@@ -220,8 +254,9 @@ def bounds(shape: Shape, measure: Measure = rough_measure) -> Box:r = shape.sizex, y = shape.points[0]return (x - r, y - r, 2 * r, 2 * r)- xs = [p[0] for p in shape.points]- ys = [p[1] for p in shape.points]+ pts = segment_path(shape) if shape.kind in SEGMENT_KINDS else shape.points+ xs = [p[0] for p in pts]+ ys = [p[1] for p in pts]return (min(xs), min(ys), max(xs) - min(xs), max(ys) - min(ys))@@ -251,7 +286,7 @@ def hit(shape: Shape, p: Point, slop: float = 6.0, measure: Measure = rough_measreturn Falseif shape.kind in SEGMENT_KINDS or shape.kind in STROKE_KINDS:reach = slop + (shape.width * 2.0 if shape.kind == HIGHLIGHT else shape.width)- pts = shape.points+ pts = segment_path(shape) if shape.kind in SEGMENT_KINDS else shape.pointsif len(pts) == 1:return math.hypot(p[0] - pts[0][0], p[1] - pts[0][1]) <= reachpairs = zip(pts, pts[1:], strict=False)@@ -294,6 +329,8 @@ def shape_at(def translated(shape: Shape, dx: float, dy: float) -> Shape:moved = shape.copy()moved.points = [(x + dx, y + dy) for x, y in shape.points]+ if shape.bend is not None:+ moved.bend = (shape.bend[0] + dx, shape.bend[1] + dy)return moved@@ -334,13 +371,18 @@ def _corner(box: Box, name: str) -> Point:def grips(shape: Shape, measure: Measure = rough_measure) -> list[tuple[str, Point]]:"""The grips a selected marking shows.- Lines and arrows one at each end; boxes, strokes and text one at each- corner (text scales with them); a counter one on its rim.+ Lines and arrows one at each end and one in the middle that bends them;+ boxes, strokes and text one at each corner (text scales with them); a+ counter one on its rim."""if not shape.points:return []if shape.kind in SEGMENT_KINDS:- return [("from", shape.points[0]), ("to", shape.points[-1])]+ return [+ ("from", shape.points[0]),+ ("mid", segment_middle(shape)),+ ("to", shape.points[-1]),+ ]if shape.kind == COUNTER:x, y = shape.points[0]return [("size", (x + shape.size, y))]@@ -364,9 +406,11 @@ def reshape(original: Shape, grip: str, p: Point, measure: Measure = rough_measu"""``original`` with ``grip`` dragged to ``p``.Always computed from the marking as it was when the drag began, so a long- drag never drifts. An arrow's end follows the pointer; a box keeps the- opposite corner fixed; a stroke scales from it; text grows or shrinks- from the opposite corner, font size and all; a counter grows with its rim.+ drag never drifts. An arrow's end follows the pointer and its middle grip+ bends it into a curve through the pointer (back to straight when dropped+ on the straight line); a box keeps the opposite corner fixed; a stroke+ scales from it; text grows or shrinks from the opposite corner, font size+ and all; a counter grows with its rim."""shape = original.copy()if shape.kind in SEGMENT_KINDS:@@ -374,6 +418,14 @@ def reshape(original: Shape, grip: str, p: Point, measure: Measure = rough_measushape.points[0] = pelif grip == "to":shape.points[-1] = p+ elif grip == "mid":+ start, end = shape.points[0], shape.points[-1]+ middle = ((start[0] + end[0]) / 2, (start[1] + end[1]) / 2)+ if math.hypot(p[0] - middle[0], p[1] - middle[1]) < STRAIGHT_SNAP_PX:+ shape.bend = None+ else:+ # The curve's middle lands on the pointer.+ shape.bend = (2 * p[0] - middle[0], 2 * p[1] - middle[1])return shapeif shape.kind == COUNTER:x, y = shape.points[0]@@ -595,6 +647,7 @@ __all__ = ["clip_to","constrain","counter_size",+ "curve_at","frame_padding","grab_scope","grip_at",@@ -610,6 +663,8 @@ __all__ = ["reshape","resize","rough_measure",+ "segment_middle",+ "segment_path","shape_at","text_size","toolbar_origin",diff --git a/jarvis/appshot/picker/renderer.py b/jarvis/appshot/picker/renderer.pyindex c05bc99930..5c458944b6 100644--- a/jarvis/appshot/picker/renderer.py+++ b/jarvis/appshot/picker/renderer.py@@ -588,7 +588,7 @@ class _SelectWindow(QWidget):self._commit_typing()m = self._markupm.tool = kind- if mm.grab_scope(kind) != "any":+ if mm.grab_scope(kind) == "none":m.selected = Noneif kind == mm.BACKGROUND and m.background is None:m.background = dict(mm.DEFAULT_BACKGROUND)@@ -743,11 +743,12 @@ class _SelectWindow(QWidget):returnshape.text = shape.text.rstrip()if mm.is_meaningful(shape):- if index is not None:- m.shapes.insert(min(index, len(m.shapes)), shape)- else:- m.shapes.append(shape)+ at = min(index, len(m.shapes)) if index is not None else len(m.shapes)+ m.shapes.insert(at, shape)+ # A finished text shows its points, like every fresh marking.+ m.selected = atself._commit()+ self._sync_options()elif index is not None:self._commit() # emptied: the text is goneself.update()@@ -762,12 +763,12 @@ class _SelectWindow(QWidget):"""The marking a press at ``point`` would move with the current tool."""m = self._markupscope = mm.grab_scope(m.tool)- if scope == "none":+ if scope in ("none", "grips"):return None- index = mm.shape_at(m.shapes, point, annotate.qt_measure, areas=m.tool == mm.MOVE)+ index = mm.shape_at(m.shapes, point, annotate.qt_measure, areas=scope == "any")if index is None or (m.tool == mm.TEXT and m.shapes[index].kind == mm.TEXT):return None- return index if scope == "any" or index == m.selected else None+ return indexdef _markup_key(self, event) -> None: # noqa: C901 - one key tablem = self._markup@@ -899,6 +900,7 @@ class _SelectWindow(QWidget):number=mm.next_counter(m.shapes),))+ m.selected = len(m.shapes) - 1self._commit()elif m.tool in mm.STROKE_KINDS:m.drawing = mm.Shape(m.tool, color=m.color, width=m.width, points=[point])@@ -964,11 +966,10 @@ class _SelectWindow(QWidget):if shape is not None and mm.is_meaningful(shape):m.shapes.append(shape)self._commit()- # A fresh shape is selected at once, so its grips are right there;- # ink is not, or its box would catch the next stroke beside it.- if mm.grab_scope(m.tool) != "none":- m.selected = len(m.shapes) - 1- self._sync_options()+ # Every fresh marking shows its points at once (the previous one+ # loses them), so it can be moved and reshaped right away.+ m.selected = len(m.shapes) - 1+ self._sync_options()self._update_markup_area()def mouseDoubleClickEvent(self, event) -> None: # noqa: N802diff --git a/tests/unit/appshot/test_appshot_markup.py b/tests/unit/appshot/test_appshot_markup.pyindex 07750adaf4..d957a4c4ce 100644--- a/tests/unit/appshot/test_appshot_markup.py+++ b/tests/unit/appshot/test_appshot_markup.py@@ -169,8 +169,24 @@ def test_grips_follow_the_full_editor_rules() -> None:assert [name for name, _ in mm.grips(counter)] == ["size"]assert mm.reshape(counter, "size", (30, 40)).size == 50assert mm.grab_scope(mm.MOVE) == "any"- assert mm.grab_scope(mm.RECT) == "selected"- assert mm.grab_scope(mm.PEN) == "none"+ assert mm.grab_scope(mm.RECT) == "shapes"+ assert mm.grab_scope(mm.COUNTER) == "shapes"+ assert mm.grab_scope(mm.PEN) == "grips"+ assert mm.grab_scope(mm.BACKGROUND) == "none"+++def test_the_middle_grip_bends_an_arrow_and_straightens_it_again() -> None:+ arrow = mm.Shape(mm.ARROW, width=3, points=[(0, 0), (100, 0)])+ assert [name for name, _ in mm.grips(arrow)] == ["from", "mid", "to"]+ curved = mm.reshape(arrow, "mid", (50, 40))+ # The curve's middle runs through the pointer.+ assert curved.bend == (50, 80)+ assert mm.segment_middle(curved) == (50, 40)+ assert mm.hit(curved, (50, 40)) and not mm.hit(curved, (50, 2))+ assert mm.bounds(curved)[3] == 40+ moved = mm.translated(curved, 10, 5)+ assert moved.bend == (60, 85)+ assert mm.reshape(curved, "mid", (51, 1)).bend is Nonedef test_areas_are_only_picked_when_nothing_on_them_is_hit() -> None: