Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

6.4 KB
Raw Blame
#!/bin/bash
# screenshot-annotate-basic.sh — annotated screenshot pipeline.
#
# Covers the public contract of /screenshot?annotate=true:
#   - JSON envelope shape (format/base64/annotations)
#   - per-annotation fields (ref, role, box.x/y/w/h with positive size)
#   - selector clip projection (boxes are clip-relative, not viewport)
#   - cleanup of the in-page __pinchtab_annotations__ overlay
#   - returned ref is usable for follow-up actions in the same tab

GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${GROUP_DIR}/../../helpers/api.sh"

# ─────────────────────────────────────────────────────────────────
start_test "screenshot annotate: JSON envelope shape"

pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/form.html\"}"

pt_get "/screenshot?annotate=true&format=png"
assert_ok "annotated screenshot"

assert_result_eq '.format' 'png' 'envelope.format = "png"'
assert_result_exists '.base64'      'envelope.base64 present'
assert_result_exists '.annotations' 'envelope.annotations present'
assert_json_length_gte "$RESULT" '.annotations' 1 'has at least one annotation'

end_test

# ─────────────────────────────────────────────────────────────────
start_test "screenshot annotate: annotation fields"

# Load the fixture directly so this check is independent from the previous
# annotated capture. Some browser providers can defer compositor work after a
# capture; this test only owns the annotation-field contract.
pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/form.html\"}"
assert_ok "navigate to form fixture for field check"

pt_get "/screenshot?annotate=true&format=png"
assert_ok "annotated screenshot for field check"

FIRST_REF=$(echo "$RESULT" | jq -r '.annotations[0].ref')

assert_result_jq \
  '.annotations[0].ref | test("^[a-z]+[0-9]+$")' \
  'annotations[0].ref looks like e<digits>' \
  'annotations[0].ref is malformed'

# Every box must have positive width and height; otherwise the agent has no
# usable target. Stricter than "field present".
assert_result_jq \
  'all(.annotations[]; .box.w > 0 and .box.h > 0)' \
  'all annotation boxes have positive size' \
  'at least one annotation has zero/negative box size'

end_test

# ─────────────────────────────────────────────────────────────────
start_test "screenshot annotate: overlay node removed after capture"

# The injector adds a single id="__pinchtab_annotations__" root and the
# orchestrator removes it on the way out. After a successful capture the
# node must not be present in the live DOM, otherwise it would leak ink
# into subsequent (non-annotated) screenshots and clicks.
pt_post /evaluate -d '{"expression":"document.getElementById(\"__pinchtab_annotations__\") === null"}'
assert_ok "evaluate overlay-removed probe"
assert_result_eq '.result' 'true' 'overlay node removed after capture'

end_test

# ─────────────────────────────────────────────────────────────────
start_test "screenshot annotate: returned ref is clickable"

# A non-annotated snapshot is intentionally NOT taken between the annotate
# call and the click — annotated capture should refresh the tab's ref map
# itself so follow-up actions resolve immediately.
pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/form.html\"}"
pt_get "/screenshot?annotate=true&format=png"
assert_ok "annotated screenshot before click"

# Pick the first annotation that points at an interactive button or link;
# fall back to the first ref if none qualify.
TARGET_REF=$(echo "$RESULT" | jq -r '
  (.annotations[] | select(.role == "button" or .role == "link") | .ref) // .annotations[0].ref
' | head -n1)

if [ -n "$TARGET_REF" ] && [ "$TARGET_REF" != "null" ]; then
  pt_post /action -d "{\"kind\":\"click\",\"ref\":\"${TARGET_REF}\"}"
  # Some refs may target submit-only elements that error out for benign
  # reasons; we only require that the ref itself was recognised, i.e. the
  # response is not "ref not found".
  if echo "$RESULT" | grep -qi "ref.*not.*found\|unknown.*ref"; then
    fail_assert "ref ${TARGET_REF} not recognised after annotate"
  else
    pass_assert "ref ${TARGET_REF} resolved by click handler"
  fi
else
  fail_assert "no annotations returned to click"
fi

end_test

# ─────────────────────────────────────────────────────────────────
start_test "screenshot annotate: selector clip projects boxes to clip origin"

pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/form.html\"}"

# The fixture form has id="test-form" (see tests/e2e/fixtures/form.html).
# Selector + annotate must clip the screenshot AND narrow the annotation
# set to refs whose rect overlaps that clip. We assert this directly:
# the call must succeed, must return at least one annotation, and every
# returned box must be clip-relative (x >= 0 and y >= 0).
SELECTOR_ENC='%23test-form'
pt_get "/screenshot?annotate=true&format=png&selector=${SELECTOR_ENC}"
assert_ok "annotated selector screenshot"

assert_result_exists '.annotations' 'envelope.annotations present (selector mode)'
assert_json_length_gte "$RESULT" '.annotations' 1 'selector returned at least one annotation'

# Clip-relative invariant: after subtracting target origin, every box must
# start at non-negative coords. A regression that left boxes in viewport
# space would surface here as negative or far-from-zero values.
assert_result_jq \
  'all(.annotations[]; .box.x >= 0 and .box.y >= 0)' \
  'selector annotations are clip-relative (all x,y >= 0)' \
  'selector annotation projection emitted negative coords'

# Sanity: at least one annotation must sit close to the clip origin
# (within 200px). If projection were skipped, the smallest box origin
# would still be at the form's viewport y, well above 200.
assert_result_jq \
  '[.annotations[] | select(.box.y < 200)] | length > 0' \
  'at least one annotation near clip origin' \
  'no annotation near clip origin — projection looks unprojected'

end_test