Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

feat(appshot): bendable arrows and points on every fresh marking in the area picker

rubenluetke10-beep387cde34a162
diff --git a/jarvis/appshot/picker/annotate.py b/jarvis/appshot/picker/annotate.py
index c357114651..5357768ac2 100644
--- a/jarvis/appshot/picker/annotate.py
+++ b/jarvis/appshot/picker/annotate.py
@@ -244,8 +244,58 @@ def qt_measure(text: str, size: float) -> float:
# --------------------------------------------------------------------------
-def tapered_arrow_outline(start: mm.Point, end: mm.Point, width: float) -> list[mm.Point]:
+def _curved_outline(start: mm.Point, end: mm.Point, width: float, bend: mm.Point) -> list[mm.Point]:
+ """The tapered arrow along a curve: the same profile, following the bend."""
+ path = mm.segment_path(mm.Shape(mm.ARROW, points=[start, end], bend=bend), steps=48)
+ lengths = [0.0]
+ for a, b in zip(path, path[1:], strict=False):
+ lengths.append(lengths[-1] + math.hypot(b[0] - a[0], b[1] - a[1]))
+ length = lengths[-1]
+ if length < 2:
+ return []
+ head = min(max(12.0, width * 4.6 + 6.0), length * 0.62)
+ barb = head * 0.56
+ neck_at = length - head * 0.7
+ tail = max(0.6, width * 0.14)
+ neck = max(1.4, width * 0.78)
+
+ def normal(i: int) -> tuple[float, float]:
+ a = path[max(0, i - 1)]
+ b = path[min(len(path) - 1, i + 1)]
+ dx, dy = b[0] - a[0], b[1] - a[1]
+ span = math.hypot(dx, dy) or 1.0
+ return (-dy / span, dx / span)
+
+ left: list[mm.Point] = []
+ right: list[mm.Point] = []
+ for i, (point, along) in enumerate(zip(path, lengths, strict=False)):
+ if along > neck_at:
+ break
+ half = tail + (neck - tail) * (along / max(1.0, neck_at))
+ nx, ny = normal(i)
+ left.append((point[0] + nx * half, point[1] + ny * half))
+ right.append((point[0] - nx * half, point[1] - ny * half))
+ # The head points along the curve's last stretch.
+ tx, ty = end[0] - bend[0], end[1] - bend[1]
+ span = math.hypot(tx, ty) or 1.0
+ ux, uy = tx / span, ty / span
+ nx, ny = -uy, ux
+ base = (end[0] - ux * head, end[1] - uy * head)
+ return [
+ *left,
+ (base[0] + nx * barb, base[1] + ny * barb),
+ end,
+ (base[0] - nx * barb, base[1] - ny * barb),
+ *reversed(right),
+ ]
+
+
+def tapered_arrow_outline(
+ start: mm.Point, end: mm.Point, width: float, bend: mm.Point | None = None
+) -> list[mm.Point]:
"""The tapered arrow, tail to tip and back (``taperedArrowOutline`` in TS)."""
+ if bend is not None:
+ return _curved_outline(start, end, width, bend)
dx, dy = end[0] - start[0], end[1] - start[1]
length = math.hypot(dx, dy)
if length < 2:
@@ -293,7 +343,7 @@ def _soft_shadow(painter: QPainter, path: QPainterPath, blur: float, dy: float,
def _tapered_arrow(painter: QPainter, shape: mm.Shape, colour: QColor) -> None:
- outline = tapered_arrow_outline(shape.points[0], shape.points[-1], shape.width)
+ outline = tapered_arrow_outline(shape.points[0], shape.points[-1], shape.width, shape.bend)
if not outline:
return
path = QPainterPath()
@@ -307,8 +357,46 @@ def _tapered_arrow(painter: QPainter, shape: mm.Shape, colour: QColor) -> None:
painter.drawPath(path)
+def _classic_head(painter: QPainter, tip: mm.Point, toward: mm.Point, shape: mm.Shape) -> None:
+ """A filled head at ``tip``, pointing away from ``toward``."""
+ colour = _qcolor(shape.color)
+ angle = math.atan2(tip[1] - toward[1], tip[0] - toward[0])
+ head = max(10.0, shape.width * 4.0)
+ painter.setPen(Qt.PenStyle.NoPen)
+ painter.setBrush(colour)
+ painter.drawPolygon(
+ _polygon(
+ [
+ tip,
+ (tip[0] - math.cos(angle - 0.45) * head, tip[1] - math.sin(angle - 0.45) * head),
+ (tip[0] - math.cos(angle + 0.45) * head, tip[1] - math.sin(angle + 0.45) * head),
+ ]
+ )
+ )
+
+
+def _segment_qpath(shape: mm.Shape) -> QPainterPath:
+ start, end = shape.points[0], shape.points[-1]
+ path = QPainterPath(QPointF(*start))
+ if shape.bend is None:
+ path.lineTo(QPointF(*end))
+ else:
+ path.quadTo(QPointF(*shape.bend), QPointF(*end))
+ return path
+
+
def _classic_arrow(painter: QPainter, start: mm.Point, end: mm.Point, shape: mm.Shape) -> None:
colour = _qcolor(shape.color)
+ if shape.bend is not None:
+ pen = QPen(colour, shape.width)
+ pen.setCapStyle(Qt.PenCapStyle.RoundCap)
+ painter.setPen(pen)
+ painter.setBrush(Qt.BrushStyle.NoBrush)
+ painter.drawPath(_segment_qpath(shape))
+ _classic_head(painter, end, shape.bend, shape)
+ if shape.style == "double":
+ _classic_head(painter, start, shape.bend, shape)
+ return
angle = math.atan2(end[1] - start[1], end[0] - start[0])
head = max(10.0, shape.width * 4.0)
back = (end[0] - math.cos(angle) * head * 0.8, end[1] - math.sin(angle) * head * 0.8)
@@ -393,10 +481,11 @@ def paint_shape(
pen = QPen(colour, shape.width)
pen.setCapStyle(Qt.PenCapStyle.RoundCap)
painter.setPen(pen)
- painter.drawLine(QPointF(*shape.points[0]), QPointF(*shape.points[-1]))
+ painter.setBrush(Qt.BrushStyle.NoBrush)
+ painter.drawPath(_segment_qpath(shape))
elif kind == mm.ARROW:
start, end = shape.points[0], shape.points[-1]
- if shape.style == "classic":
+ if shape.style == "classic" or (shape.style == "double" and shape.bend is not None):
_classic_arrow(painter, start, end, shape)
elif shape.style == "double":
_classic_arrow(painter, start, end, shape)
@@ -539,10 +628,11 @@ def paint_selection_frame(painter: QPainter, sel: QRectF) -> None:
def paint_picked(painter: QPainter, shape: mm.Shape) -> None:
- """The selected marking: a dashed frame (not for lines) and its grips."""
+ """The selected marking's points, plus a dashed frame where the points alone
+ would float (text, strokes)."""
painter.save()
painter.setRenderHint(QPainter.RenderHint.Antialiasing)
- if shape.kind not in mm.SEGMENT_KINDS:
+ if shape.kind == mm.TEXT or shape.kind in mm.STROKE_KINDS:
box = _qrect(mm.bounds(shape, qt_measure)).adjusted(-6, -6, 6, 6)
painter.setBrush(Qt.BrushStyle.NoBrush)
dark = QPen(QColor(0, 0, 0, 150), 1.5, Qt.PenStyle.DashLine)
@@ -1147,8 +1237,16 @@ class Toolbar(QFrame):
button.setToolTip(self._labels.get(kind, value))
button.clicked.connect(lambda _c=False, g=group, v=value: self.option_chosen.emit(g, v))
self._options.addWidget(button)
+ # Shown now, not on the layout's queued show: a hidden button does
+ # not count when the bar measures itself, and the new looks then
+ # pile up on top of each other and the colours.
+ button.show()
self._option_buttons.append(button)
self._optsep.setVisible(bool(self._option_buttons))
+ layout = self.layout()
+ if layout is not None:
+ layout.invalidate()
+ layout.activate()
self.adjustSize()
diff --git a/jarvis/appshot/picker/markup_model.py b/jarvis/appshot/picker/markup_model.py
index 592f4d445c..24d23c4e62 100644
--- a/jarvis/appshot/picker/markup_model.py
+++ b/jarvis/appshot/picker/markup_model.py
@@ -110,6 +110,8 @@ HANDLE_GRAB_PX = 8.0
MIN_SELECTION_PX = 8.0
#: The smallest text and counter a grip can shrink them to (px).
MIN_TEXT_PX = 8.0
+#: A bend grip dropped this close to the straight line straightens it again.
+STRAIGHT_SNAP_PX = 4.0
@dataclass(slots=True)
@@ -119,7 +121,8 @@ class Shape:
box kinds and segments: ``[start, end]``; strokes: the whole path; text:
``[top-left]``; counter: ``[centre]``. ``size`` is the text height or the
counter's radius; ``style`` the arrow or text look; ``mode`` pixelate or
- blur for a redaction.
+ blur for a redaction. ``bend`` is the control point of a curved arrow or
+ line (a quadratic curve from start to end); ``None`` keeps it straight.
"""
kind: str
@@ -131,6 +134,7 @@ class Shape:
size: float = 0.0
style: str = ""
mode: str = ""
+ bend: Point | None = None
def copy(self) -> Shape:
return replace(self, points=list(self.points))
@@ -158,15 +162,20 @@ def counter_size(width: float) -> float:
def grab_scope(tool: str) -> str:
"""What a press with ``tool`` may take hold of instead of drawing.
- ``any``: the select tool picks up every marking; ``selected``: a shape
- tool reshapes or moves only the marking it has selected (the one it just
- drew); ``none``: pen, highlighter and counter always draw.
+ ``any``: the select tool picks up every marking, areas included;
+ ``shapes``: every other drawing tool picks up any marking under the
+ pointer (not areas — they cover what is drawn in them), so anything
+ drawn can be moved and reshaped again at once; ``grips``: pen and
+ highlighter only take the selected marking's points, so writing over
+ earlier ink never drags it along; ``none``: the background switch.
"""
if tool == MOVE:
return "any"
- if tool in (PEN, HIGHLIGHT, COUNTER, BACKGROUND):
+ if tool in (PEN, HIGHLIGHT):
+ return "grips"
+ if tool == BACKGROUND:
return "none"
- return "selected"
+ return "shapes"
# --------------------------------------------------------------------------
@@ -197,6 +206,31 @@ def constrain(kind: str, start: Point, end: Point) -> Point:
return end
+def curve_at(start: Point, bend: Point, end: Point, t: float) -> Point:
+ """The point at ``t`` on the quadratic curve start - bend - end."""
+ u = 1.0 - t
+ return (
+ u * u * start[0] + 2 * u * t * bend[0] + t * t * end[0],
+ u * u * start[1] + 2 * u * t * bend[1] + t * t * end[1],
+ )
+
+
+def segment_path(shape: Shape, steps: int = 32) -> list[Point]:
+ """An arrow's or line's path as points: the two ends, or the sampled curve."""
+ start, end = shape.points[0], shape.points[-1]
+ if shape.bend is None:
+ return [start, end]
+ return [curve_at(start, shape.bend, end, i / steps) for i in range(steps + 1)]
+
+
+def segment_middle(shape: Shape) -> Point:
+ """Where the bend grip sits: the middle of the line or of the curve."""
+ start, end = shape.points[0], shape.points[-1]
+ if shape.bend is None:
+ return ((start[0] + end[0]) / 2, (start[1] + end[1]) / 2)
+ return curve_at(start, shape.bend, end, 0.5)
+
+
def label_pad(shape: Shape) -> float:
return shape.size * 0.4 if shape.kind == TEXT and shape.style == "label" else 0.0
@@ -220,8 +254,9 @@ def bounds(shape: Shape, measure: Measure = rough_measure) -> Box:
r = shape.size
x, y = shape.points[0]
return (x - r, y - r, 2 * r, 2 * r)
- xs = [p[0] for p in shape.points]
- ys = [p[1] for p in shape.points]
+ pts = segment_path(shape) if shape.kind in SEGMENT_KINDS else shape.points
+ xs = [p[0] for p in pts]
+ ys = [p[1] for p in pts]
return (min(xs), min(ys), max(xs) - min(xs), max(ys) - min(ys))
@@ -251,7 +286,7 @@ def hit(shape: Shape, p: Point, slop: float = 6.0, measure: Measure = rough_meas
return False
if shape.kind in SEGMENT_KINDS or shape.kind in STROKE_KINDS:
reach = slop + (shape.width * 2.0 if shape.kind == HIGHLIGHT else shape.width)
- pts = shape.points
+ pts = segment_path(shape) if shape.kind in SEGMENT_KINDS else shape.points
if len(pts) == 1:
return math.hypot(p[0] - pts[0][0], p[1] - pts[0][1]) <= reach
pairs = zip(pts, pts[1:], strict=False)
@@ -294,6 +329,8 @@ def shape_at(
def translated(shape: Shape, dx: float, dy: float) -> Shape:
moved = shape.copy()
moved.points = [(x + dx, y + dy) for x, y in shape.points]
+ if shape.bend is not None:
+ moved.bend = (shape.bend[0] + dx, shape.bend[1] + dy)
return moved
@@ -334,13 +371,18 @@ def _corner(box: Box, name: str) -> Point:
def grips(shape: Shape, measure: Measure = rough_measure) -> list[tuple[str, Point]]:
"""The grips a selected marking shows.
- Lines and arrows one at each end; boxes, strokes and text one at each
- corner (text scales with them); a counter one on its rim.
+ Lines and arrows one at each end and one in the middle that bends them;
+ boxes, strokes and text one at each corner (text scales with them); a
+ counter one on its rim.
"""
if not shape.points:
return []
if shape.kind in SEGMENT_KINDS:
- return [("from", shape.points[0]), ("to", shape.points[-1])]
+ return [
+ ("from", shape.points[0]),
+ ("mid", segment_middle(shape)),
+ ("to", shape.points[-1]),
+ ]
if shape.kind == COUNTER:
x, y = shape.points[0]
return [("size", (x + shape.size, y))]
@@ -364,9 +406,11 @@ def reshape(original: Shape, grip: str, p: Point, measure: Measure = rough_measu
"""``original`` with ``grip`` dragged to ``p``.
Always computed from the marking as it was when the drag began, so a long
- drag never drifts. An arrow's end follows the pointer; a box keeps the
- opposite corner fixed; a stroke scales from it; text grows or shrinks
- from the opposite corner, font size and all; a counter grows with its rim.
+ drag never drifts. An arrow's end follows the pointer and its middle grip
+ bends it into a curve through the pointer (back to straight when dropped
+ on the straight line); a box keeps the opposite corner fixed; a stroke
+ scales from it; text grows or shrinks from the opposite corner, font size
+ and all; a counter grows with its rim.
"""
shape = original.copy()
if shape.kind in SEGMENT_KINDS:
@@ -374,6 +418,14 @@ def reshape(original: Shape, grip: str, p: Point, measure: Measure = rough_measu
shape.points[0] = p
elif grip == "to":
shape.points[-1] = p
+ elif grip == "mid":
+ start, end = shape.points[0], shape.points[-1]
+ middle = ((start[0] + end[0]) / 2, (start[1] + end[1]) / 2)
+ if math.hypot(p[0] - middle[0], p[1] - middle[1]) < STRAIGHT_SNAP_PX:
+ shape.bend = None
+ else:
+ # The curve's middle lands on the pointer.
+ shape.bend = (2 * p[0] - middle[0], 2 * p[1] - middle[1])
return shape
if shape.kind == COUNTER:
x, y = shape.points[0]
@@ -595,6 +647,7 @@ __all__ = [
"clip_to",
"constrain",
"counter_size",
+ "curve_at",
"frame_padding",
"grab_scope",
"grip_at",
@@ -610,6 +663,8 @@ __all__ = [
"reshape",
"resize",
"rough_measure",
+ "segment_middle",
+ "segment_path",
"shape_at",
"text_size",
"toolbar_origin",
diff --git a/jarvis/appshot/picker/renderer.py b/jarvis/appshot/picker/renderer.py
index c05bc99930..5c458944b6 100644
--- a/jarvis/appshot/picker/renderer.py
+++ b/jarvis/appshot/picker/renderer.py
@@ -588,7 +588,7 @@ class _SelectWindow(QWidget):
self._commit_typing()
m = self._markup
m.tool = kind
- if mm.grab_scope(kind) != "any":
+ if mm.grab_scope(kind) == "none":
m.selected = None
if kind == mm.BACKGROUND and m.background is None:
m.background = dict(mm.DEFAULT_BACKGROUND)
@@ -743,11 +743,12 @@ class _SelectWindow(QWidget):
return
shape.text = shape.text.rstrip()
if mm.is_meaningful(shape):
- if index is not None:
- m.shapes.insert(min(index, len(m.shapes)), shape)
- else:
- m.shapes.append(shape)
+ at = min(index, len(m.shapes)) if index is not None else len(m.shapes)
+ m.shapes.insert(at, shape)
+ # A finished text shows its points, like every fresh marking.
+ m.selected = at
self._commit()
+ self._sync_options()
elif index is not None:
self._commit() # emptied: the text is gone
self.update()
@@ -762,12 +763,12 @@ class _SelectWindow(QWidget):
"""The marking a press at ``point`` would move with the current tool."""
m = self._markup
scope = mm.grab_scope(m.tool)
- if scope == "none":
+ if scope in ("none", "grips"):
return None
- index = mm.shape_at(m.shapes, point, annotate.qt_measure, areas=m.tool == mm.MOVE)
+ index = mm.shape_at(m.shapes, point, annotate.qt_measure, areas=scope == "any")
if index is None or (m.tool == mm.TEXT and m.shapes[index].kind == mm.TEXT):
return None
- return index if scope == "any" or index == m.selected else None
+ return index
def _markup_key(self, event) -> None: # noqa: C901 - one key table
m = self._markup
@@ -899,6 +900,7 @@ class _SelectWindow(QWidget):
number=mm.next_counter(m.shapes),
)
)
+ m.selected = len(m.shapes) - 1
self._commit()
elif m.tool in mm.STROKE_KINDS:
m.drawing = mm.Shape(m.tool, color=m.color, width=m.width, points=[point])
@@ -964,11 +966,10 @@ class _SelectWindow(QWidget):
if shape is not None and mm.is_meaningful(shape):
m.shapes.append(shape)
self._commit()
- # A fresh shape is selected at once, so its grips are right there;
- # ink is not, or its box would catch the next stroke beside it.
- if mm.grab_scope(m.tool) != "none":
- m.selected = len(m.shapes) - 1
- self._sync_options()
+ # Every fresh marking shows its points at once (the previous one
+ # loses them), so it can be moved and reshaped right away.
+ m.selected = len(m.shapes) - 1
+ self._sync_options()
self._update_markup_area()
def mouseDoubleClickEvent(self, event) -> None: # noqa: N802
diff --git a/tests/unit/appshot/test_appshot_markup.py b/tests/unit/appshot/test_appshot_markup.py
index 07750adaf4..d957a4c4ce 100644
--- a/tests/unit/appshot/test_appshot_markup.py
+++ b/tests/unit/appshot/test_appshot_markup.py
@@ -169,8 +169,24 @@ def test_grips_follow_the_full_editor_rules() -> None:
assert [name for name, _ in mm.grips(counter)] == ["size"]
assert mm.reshape(counter, "size", (30, 40)).size == 50
assert mm.grab_scope(mm.MOVE) == "any"
- assert mm.grab_scope(mm.RECT) == "selected"
- assert mm.grab_scope(mm.PEN) == "none"
+ assert mm.grab_scope(mm.RECT) == "shapes"
+ assert mm.grab_scope(mm.COUNTER) == "shapes"
+ assert mm.grab_scope(mm.PEN) == "grips"
+ assert mm.grab_scope(mm.BACKGROUND) == "none"
+
+
+def test_the_middle_grip_bends_an_arrow_and_straightens_it_again() -> None:
+ arrow = mm.Shape(mm.ARROW, width=3, points=[(0, 0), (100, 0)])
+ assert [name for name, _ in mm.grips(arrow)] == ["from", "mid", "to"]
+ curved = mm.reshape(arrow, "mid", (50, 40))
+ # The curve's middle runs through the pointer.
+ assert curved.bend == (50, 80)
+ assert mm.segment_middle(curved) == (50, 40)
+ assert mm.hit(curved, (50, 40)) and not mm.hit(curved, (50, 2))
+ assert mm.bounds(curved)[3] == 40
+ moved = mm.translated(curved, 10, 5)
+ assert moved.bend == (60, 85)
+ assert mm.reshape(curved, "mid", (51, 1)).bend is None
def test_areas_are_only_picked_when_nothing_on_them_is_hit() -> None: