Skip to content

Realistic watercolor drawings

TypeScript paintwatercolor
7.4 KB
Raw Blame
import type { NormalizedLandmark } from '@mediapipe/tasks-vision';
import { Dwell, sliderInput, toRect } from './dwell';
import { Ema, Hysteresis, OneEuro } from './filter';
import { HandTracker, type Frame } from './tracker';
import { HAND_IDLE, type HandHost, type HandRect } from './types';

/** Rest on a control this long to press it. */
const DWELL_MS = 1500;
/** Pressure of every hand stroke (there is no stylus to read it from). */
const PRESSURE = 0.6;
/** The part of the camera frame that maps to the whole screen, so the corners need no reaching. */
const BOX_LO = 0.15, BOX_HI = 0.85;
/** How long the cursor stays where it was once the hand is lost. */
const LOST_MS = 150;
/** Pinch thresholds on thumb-tip to index-tip distance over palm size (wrist to middle knuckle). */
const PINCH_ENTER = 0.35, PINCH_EXIT = 0.5;
/** How far up or down the hand may wander while sliding before the slider is let go, in row heights. */
const GRAB_LEEWAY = 2.5;

const THUMB_TIP = 4, INDEX_TIP = 8, WRIST = 0, MIDDLE_MCP = 9;
const clamp = (v: number, lo: number, hi: number) => Math.min(hi, Math.max(lo, v));

interface Grab { input: HTMLInputElement; track: HandRect; row: HandRect; last: number }

/**
 * Paints and operates the UI from camera hand landmarks: the midpoint of thumb and index
 * tip is the cursor, a pinch is the brush on the paper, resting on a control presses it,
 * and resting on a slider grabs it so sideways motion sets its value.
 */
export class HandMode {
  private readonly tracker: HandTracker;
  private readonly fx = new OneEuro();
  private readonly fy = new OneEuro();
  private readonly pinchRatio = new Ema(0.5);
  private readonly pinch = new Hysteresis(PINCH_ENTER, PINCH_EXIT);
  private readonly dwell = new Dwell(DWELL_MS);
  private mode: 'idle' | 'paint' | 'grab' = 'idle';
  private grab: Grab | null = null;
  private wasPinching = false;
  private seenAt = -Infinity;
  private stopped = false;

  constructor(private host: HandHost) {
    this.tracker = new HandTracker(f => this.onFrame(f), s => host.setStream(s));
  }

  async start(): Promise<void> {
    document.addEventListener('visibilitychange', this.onVisibility);
    try {
      await this.tracker.start();
    } catch (e) {
      this.stop();
      throw e;
    }
  }

  stop() {
    if (this.stopped) return;
    this.stopped = true;
    document.removeEventListener('visibilitychange', this.onVisibility);
    this.endStroke();
    this.tracker.stop();
    this.host.setStream(null);
    this.host.setCursor(HAND_IDLE);
  }

  /** Feed a frame by hand (tests, no camera): 21 landmarks in camera space, or null for "no hand". */
  inject(landmarks: NormalizedLandmark[] | null, t = performance.now(), aspect = 4 / 3) {
    this.onFrame({ landmarks, t, aspect });
  }

  private onVisibility = () => {
    if (document.visibilityState !== 'hidden') return;
    this.rest();
    this.fx.reset(); this.fy.reset(); this.pinchRatio.reset(); this.pinch.reset();
    this.seenAt = -Infinity;
    this.host.setCursor(HAND_IDLE);
  };

  /** Let go of everything: the stroke, the slider, the dwell. */
  private rest() {
    this.endStroke();
    this.releaseGrab();
    this.dwell.reset();
    this.mode = 'idle';
    this.wasPinching = false;
  }

  private endStroke() {
    if (this.host.engine.scripting) this.host.engine.scriptEnd();
    if (this.mode === 'paint') this.mode = 'idle';
  }

  private releaseGrab() {
    const g = this.grab;
    if (!g) return;
    this.grab = null;
    // sliders that record an undo step per adjustment listen for `change`
    g.input.dispatchEvent(new Event('change', { bubbles: true }));
    if (this.mode === 'grab') this.mode = 'idle';
  }

  private onFrame(f: Frame) {
    if (this.stopped) return;
    const lm = f.landmarks;
    if (!lm) {
      if (f.t - this.seenAt > LOST_MS) {
        this.rest();
        this.host.setCursor(HAND_IDLE);
      }
      return;
    }
    this.seenAt = f.t;

    // the cursor: between thumb and index tip, mirrored, the middle of the frame spread over the screen
    const px = (lm[THUMB_TIP].x + lm[INDEX_TIP].x) / 2, py = (lm[THUMB_TIP].y + lm[INDEX_TIP].y) / 2;
    const u = this.fx.filter(1 - px, f.t), v = this.fy.filter(py, f.t);
    const x = clamp((u - BOX_LO) / (BOX_HI - BOX_LO), 0, 1) * window.innerWidth;
    const y = clamp((v - BOX_LO) / (BOX_HI - BOX_LO), 0, 1) * window.innerHeight;

    // the pinch: tip distance relative to the palm, so it works at any distance from the camera
    const d = (i: number, j: number) => Math.hypot((lm[i].x - lm[j].x) * f.aspect, lm[i].y - lm[j].y);
    const ratio = this.pinchRatio.update(d(THUMB_TIP, INDEX_TIP) / Math.max(d(WRIST, MIDDLE_MCP), 1e-3));
    const pinching = this.pinch.update(ratio);
    const rose = pinching && !this.wasPinching, fell = !pinching && this.wasPinching;
    this.wasPinching = pinching;

    let dwell = 0, target: HandRect | null = null;
    const engine = this.host.engine;

    if (this.mode === 'paint') {
      if (fell) {
        engine.scriptEnd();
        this.mode = 'idle';
      } else if (pinching) {
        const [dx, dy] = engine.clientToDoc(x, y);
        if (!engine.scriptMove(dx, dy, PRESSURE, f.t)) this.mode = 'idle';   // undone or cleared under us
      } else {
        this.mode = 'idle';
      }
    } else if (this.mode === 'grab') {
      const g = this.grab!;
      const rowMid = g.row.top + g.row.height / 2;
      if (pinching || Math.abs(y - rowMid) > GRAB_LEEWAY * g.row.height) {
        this.releaseGrab();
        this.dwell.reset();
      } else {
        this.slide(g, x);
        target = g.row;
      }
    }

    if (this.mode === 'idle') {
      if (pinching) {
        // a pinch over the paper starts a stroke; over a panel it is ignored (dwell is the way to press things)
        const tool = this.host.paintTool();
        const overUi = !!document.elementFromPoint(x, y)?.closest('.panel');
        if (tool && !overUi && (rose || this.wasPinching)) {
          const [dx, dy] = engine.clientToDoc(x, y);
          if (engine.scriptBegin(tool, dx, dy, PRESSURE, 'Hand stroke')) this.mode = 'paint';
          // else a pointer stroke is still on the paper: try again next frame
        }
        this.dwell.reset();
      } else {
        const r = this.dwell.update(x, y, f.t);
        dwell = r.progress; target = r.rect;
        if (r.fired?.kind === 'button') {
          r.fired.el.click();
          if (this.stopped) return;             // the hand pressed "Off"
        } else if (r.fired?.kind === 'slider') {
          const input = sliderInput(r.fired.el);
          if (input) {
            this.grab = { input, track: toRect(input.getBoundingClientRect()), row: r.rect!, last: Number(input.value) };
            this.mode = 'grab';
            this.slide(this.grab, x);
          }
        }
      }
    }

    this.host.setCursor({
      visible: true, x, y, pinching, painting: this.mode === 'paint', dwell, target,
      grab: this.mode === 'grab' ? this.grab!.row : null,
    });
  }

  /** Set a grabbed slider from the cursor's position along its track. */
  private slide(g: Grab, x: number) {
    const input = g.input;
    const min = Number(input.min) || 0, max = input.max === '' ? 1 : Number(input.max), step = Number(input.step) || 0.01;
    const f = clamp((x - g.track.left) / Math.max(g.track.width, 1), 0, 1);
    const v = clamp(min + Math.round((f * (max - min)) / step) * step, min, max);
    if (Math.abs(v - g.last) < step / 2) return;
    g.last = v;
    input.value = String(v);
    input.dispatchEvent(new Event('input', { bubbles: true }));
  }
}