Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

24.4 KB
Raw Blame
/** Runtime tier: real Electron input through cross-origin + same-origin iframes.
 * Run with node apps/desktop/scripts/native-trackpad.verify.mjs.
 * The AppKit source is substituted with recorded normalized contacts: this does
 * not claim physical-trackpad verification. --headless also substitutes focus
 * and pointer-entry signals, retaining real renderer/CDP delivery for locked
 * desktops; it does not verify OS focus. No Cindy profile is opened. */
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import http from 'node:http';
import assert from 'node:assert/strict';
import { fileURLToPath } from 'node:url';
import { createRequire } from 'node:module';
import { execFileSync } from 'node:child_process';

const require = createRequire(import.meta.url);
const desktop = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
// Optional interactive acceptance window, backed by the same production input
// module and the real addon. The URL file avoids printing viewer credentials.
const viewerFile =
  !process.versions.electron && process.argv[2] === '--viewer' ? process.argv[3] : undefined;
if (!process.versions.electron) {
  const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'cindy-trackpad-verify-'));
  try {
    require('esbuild').buildSync({
      entryPoints: [path.join(desktop, 'src/main/nativeTrackpad.ts')],
      outfile: path.join(temp, 'input.cjs'),
      bundle: true,
      platform: 'node',
      format: 'cjs',
      external: ['electron'],
    });
    const env = { ...process.env };
    env.CINDY_TRACKPAD_VERIFY_CONTACTS = process.argv.includes('--contacts') ? '1' : '0';
    env.CINDY_TRACKPAD_VERIFY_HEADLESS = process.argv.includes('--headless') ? '1' : '0';
    env.CINDY_TRACKPAD_EXERCISE = process.argv.includes('--exercise') ? '1' : '0';
    delete env.ELECTRON_RUN_AS_NODE;
    execFileSync(
      require('electron'),
      [fileURLToPath(import.meta.url), temp, ...(viewerFile ? [path.resolve(viewerFile)] : [])],
      {
        env,
        stdio: 'inherit',
        timeout: viewerFile ? undefined : 30000,
      },
    );
  } finally {
    fs.rmSync(temp, { recursive: true, force: true });
  }
} else {
  void (async () => {
    const { app, BrowserWindow } = require('electron');
    const temp = process.argv[2];
    const headless = process.env.CINDY_TRACKPAD_VERIFY_HEADLESS === '1';
    app.setPath('userData', path.join(temp, 'profile'));
    let win, server;
    const deadline = setTimeout(() => app.exit(1), 25000);
    try {
      await app.whenReady();
      if (headless) console.log('DOM_CDP_ONLY: synthetic focus, pointer entry and AppKit samples');
      const { installNativeTrackpad } = require(path.join(temp, 'input.cjs'));
      const addon = require(path.join(desktop, 'resources/tools/trackpad/cindy-trackpad.node'));
      assert.equal(typeof addon.start, 'function');
      assert.equal(typeof addon.position, 'function');
      assert.equal(typeof addon.stop, 'function');
      addon.stop(0); // Unbound/stale capture cleanup must also be callable.
      if (process.argv[3]) {
        const url = fs.readFileSync(process.argv[3], 'utf8').trim(),
          parsed = new URL(url);
        assert.equal(parsed.protocol, 'http:');
        assert(['127.0.0.1', 'localhost'].includes(parsed.hostname));
        const exercise = process.env.CINDY_TRACKPAD_EXERCISE === '1';
        if (!exercise) clearTimeout(deadline);
        let exercisePoint, exerciseInput;
        win = new BrowserWindow({
          width: 900,
          height: 1000,
          title: '设备模拟器 · 原生双指测试',
          webPreferences: { sandbox: true, contextIsolation: true },
        });
        const inputTrace = [];
        if (exercise) {
          const command = win.webContents.debugger.sendCommand.bind(win.webContents.debugger);
          win.webContents.debugger.sendCommand = async (method, ...args) => {
            try {
              const r = await command(method, ...args);
              inputTrace.push({
                method,
                node: r.node?.nodeName,
                box: r.model?.content,
                value: r.result?.value,
                object: !!r.result?.objectId,
              });
              return r;
            } catch (error) {
              inputTrace.push({ method, error: String(error) });
              throw error;
            }
          };
          win.webContents.on('before-mouse-event', (_e, input) =>
            inputTrace.push({ input: input.type }),
          );
          win.webContents.on('did-start-navigation', (_e, _url, inPlace, main) =>
            inputTrace.push({ inPlace, main }),
          );
        }
        installNativeTrackpad(win.webContents, win.webContents, () => true, {
          platform: process.platform,
          window: () => win,
          load: () => ({
            position: (handle) => (exercise ? exercisePoint : addon.position(handle)),
            stop: (id) => {
              if (!exercise) addon.stop(id);
            },
            start: (handle, callback) => {
              if (exercise) {
                exerciseInput = callback;
                return 1;
              }
              const id = addon.start(handle, (packet) => {
                const p = JSON.parse(packet);
                if (p.phase === 'begin') console.log('NATIVE_CONTACTS_BEGIN');
                callback(packet);
              });
              if (id !== null) console.log('NATIVE_CAPTURE_READY');
              return id;
            },
          }),
        });
        win.on('page-title-updated', (event) => event.preventDefault());
        app.on('window-all-closed', () => app.quit());
        await win.loadURL(url);
        win.setTitle('设备模拟器 · 原生双指测试');
        win.show();
        app.focus({ steal: true });
        win.focus();
        console.log('NATIVE_PREVIEW_READY');
        if (exercise) {
          // Explicit runtime tier: replay contacts through the actual viewer
          // and its WebSocket/device transport; never presented as hardware input.
          const find = async (frame, offset = { x: 0, y: 0 }) => {
            const r = await frame.executeJavaScript(
              `(()=>{const e=document.querySelector('[data-cindy-trackpad="contacts"]');if(!e)return null;const r=e.getBoundingClientRect();return {x:r.x,y:r.y,width:r.width,height:r.height};})()`,
            );
            if (r?.width > 0)
              return {
                frame,
                x: offset.x + r.x,
                y: offset.y + r.y,
                width: r.width,
                height: r.height,
              };
            const childBounds = await frame.executeJavaScript(
              `(()=>[...document.querySelectorAll('iframe')].map(e=>{const r=e.getBoundingClientRect();return {url:e.src,x:r.x,y:r.y};}))()`,
            );
            for (const child of frame.frames) {
              const bounds = childBounds.find((entry) => entry.url === child.url);
              if (bounds) {
                const found = await find(child, { x: offset.x + bounds.x, y: offset.y + bounds.y });
                if (found) return found;
              }
            }
          };
          let target;
          for (let i = 0; i < 100 && !target; i++) {
            target = await find(win.webContents.mainFrame);
            if (!target) await new Promise((r) => setTimeout(r, 50));
          }
          assert(target, 'Live contact surface missing');
          await target.frame.executeJavaScript(
            `window.__inputProbe={sent:[],painted:0,received:0,decoded:0,raf:0};const tick=()=>{__inputProbe.raf++;requestAnimationFrame(tick)};requestAnimationFrame(tick);if(window.VideoDecoder){const decode=VideoDecoder.prototype.decode;VideoDecoder.prototype.decode=function(...args){__inputProbe.decoded++;return decode.apply(this,args);};}const originalSend=WebSocket.prototype.send;WebSocket.prototype.send=function(data){if(!this.__probed){this.__probed=true;this.addEventListener('message',e=>{if(e.data instanceof ArrayBuffer)__inputProbe.received++;});}try{const p=JSON.parse(data);if(p.type?.startsWith('touch'))__inputProbe.sent.push({at:Date.now(),...p});}catch{}return originalSend.call(this,data);};const originalDraw=CanvasRenderingContext2D.prototype.drawImage;CanvasRenderingContext2D.prototype.drawImage=function(...args){__inputProbe.painted++;return originalDraw.apply(this,args);};`,
          );
          for (let i = 0; i < 200; i++) {
            if (await target.frame.executeJavaScript('__inputProbe.painted>=3')) break;
            await new Promise((r) => setTimeout(r, 25));
          }
          target = await find(win.webContents.mainFrame);
          assert(target, 'Live surface disappeared');
          exercisePoint = { x: target.x + target.width * 0.5, y: target.y + target.height * 0.3 };
          for (let i = 0; i < 100 && !exerciseInput; i++) {
            if (!win.isFocused()) app.focus({ steal: true });
            win.focus();
            win.webContents.focus();
            win.webContents.sendInputEvent({ type: 'mouseMove', ...exercisePoint });
            await new Promise((r) => setTimeout(r, 10));
          }
          if (!exerciseInput) {
            fs.writeFileSync(
              '/tmp/cindy-trackpad-arm-failure.png',
              (await win.webContents.capturePage()).toPNG(),
            );
            console.log(
              'ARM_FAILURE',
              JSON.stringify({
                focus: win.isFocused(),
                guestFocus: win.webContents.isFocused(),
                point: exercisePoint,
                meta: await win.webContents.executeJavaScript(
                  'document.querySelector("meta[name=cindy-native-trackpad]")?.content',
                ),
                trace: inputTrace.slice(-18),
              }),
            );
          }
          assert(exerciseInput, 'Live input not armed');
          const emit = (phase, span) =>
            exerciseInput(
              JSON.stringify({
                phase,
                width: 100,
                height: 70,
                touches:
                  phase === 'end'
                    ? []
                    : [
                        { id: 0, x: 0.5 - span, y: 0.5 },
                        { id: 1, x: 0.5 + span, y: 0.5 },
                      ],
              }),
            );
          // capturePage immediately before playback perturbs video cadence on
          // macOS (21 vs 104 painted frames in the same ~2 s replay). Capture
          // evidence only after measuring so the observer does not become the
          // apparent input/stream bottleneck.
          await target.frame.executeJavaScript(
            '__inputProbe.painted=__inputProbe.decoded=__inputProbe.received=__inputProbe.raf=0',
          );
          const started = Date.now();
          emit('begin', 0.12);
          for (let i = 0; i < 100; i++) {
            emit('change', 0.12 + i * 0.0018);
            await new Promise((r) => setTimeout(r, 8));
          }
          for (let i = 98; i >= 0; i--) {
            emit('change', 0.12 + i * 0.0018);
            await new Promise((r) => setTimeout(r, 8));
          }
          emit('end', 0);
          await new Promise((r) => setTimeout(r, 250));
          const probe = await target.frame.executeJavaScript('__inputProbe');
          console.log(
            'LIVE_INPUT',
            JSON.stringify({
              elapsed: Date.now() - started,
              frames: probe.painted,
              received: probe.received,
              decoded: probe.decoded,
              raf: probe.raf,
              types: probe.sent.reduce((a, p) => ((a[p.type] = (a[p.type] || 0) + 1), a), {}),
              first: probe.sent[0],
              last: probe.sent.at(-1),
            }),
          );
          assert.equal(probe.sent[0]?.type, 'touch2-down');
          assert.equal(probe.sent.at(-1)?.type, 'touch2-up');
          assert(!probe.sent.some((p) => p.type.startsWith('touch1')));
          assert.equal(
            probe.sent.length,
            201,
            'Replay interrupted (focus/navigation) or mixed with another input source',
          );
          assert(probe.painted >= 70, 'Full viewer video cadence is still low');
          fs.writeFileSync(
            '/tmp/cindy-trackpad-live-after.png',
            (await win.webContents.capturePage()).toPNG(),
          );
          win.destroy();
          clearTimeout(deadline);
          app.exit(0);
        }
        return;
      }
      server = http.createServer((req, res) => {
        res.setHeader('Content-Type', 'text/html');
        const port = server.address().port;
        if (req.url === '/outer')
          res.end(
            `<meta name="cindy-native-trackpad" content="touch"><iframe style="position:absolute;left:40px;top:60px;width:500px;height:600px;border:0" src="http://127.0.0.1:${port}/middle"></iframe>`,
          );
        else if (req.url === '/middle')
          res.end(
            '<iframe style="position:absolute;left:10px;top:10px;width:480px;height:580px;border:0" src="/screen"></iframe>',
          );
        else
          res.end(`<style>body{margin:0}canvas{position:absolute;left:20px;top:10px;width:400px;height:500px;touch-action:none}</style><canvas data-cindy-trackpad="${process.env.CINDY_TRACKPAD_VERIFY_CONTACTS === '1' ? 'contacts' : ''}"></canvas><script>
        window.events=[];let generation=0;
        window.bindSurface=element=>{
          element.dataset.generation=String(generation);
          for(const type of ['touchstart','touchmove','touchend','touchcancel'])element.addEventListener(type,e=>{e.preventDefault();events.push({type,at:Date.now(),target:element.dataset.generation,connected:element.isConnected,touches:[...e.touches].map(t=>({id:t.identifier,x:t.clientX,y:t.clientY}))});},{passive:false});
          let count=0;element.addEventListener('cindy-trackpad',e=>{
            const f=e.detail,type=f.phase==='begin'?'touchstart':f.phase==='cancel'?'touchcancel':f.phase==='end'||f.touches.length<count?'touchend':'touchmove';
            count=f.touches.length;events.push({type,at:Date.now(),target:element.dataset.generation,connected:element.isConnected,touches:f.touches});
          });
        };
        bindSurface(document.querySelector('canvas'));
        window.replaceSurface=cover=>{
          const old=[...document.querySelectorAll('canvas')].at(-1),next=old.cloneNode();generation++;bindSurface(next);
          if(cover)old.after(next);else old.replaceWith(next);
        };
      </script>`);
      });
      await new Promise((r) => server.listen(0, '127.0.0.1', r));
      win = new BrowserWindow({
        show: !headless,
        width: 700,
        height: 800,
        webPreferences: { sandbox: true, contextIsolation: true },
      });
      const wc = win.webContents;
      // Explicit DOM/CDP-only tier for a locked/unavailable desktop. The
      // production focus guards remain unchanged; this mode substitutes their
      // inputs, so it is not evidence of OS focus or physical AppKit capture.
      const fixtureProxy = (target, overrides) =>
        new Proxy(target, {
          get(object, key) {
            if (key in overrides) return overrides[key];
            const value = Reflect.get(object, key);
            return typeof value === 'function' ? value.bind(object) : value;
          },
        });
      const inputContents = headless ? fixtureProxy(wc, { isFocused: () => true }) : wc;
      const inputWindow = headless
        ? fixtureProxy(win, { isFocused: () => true, isVisible: () => true })
        : win;
      const commands = [];
      let pointerEvents = 0;
      wc.on('before-mouse-event', () => pointerEvents++);
      const command = wc.debugger.sendCommand.bind(wc.debugger);
      wc.debugger.sendCommand = async (method, ...args) => {
        commands.push(method);
        try {
          const result = await command(method, ...args);
          if (method === 'Input.dispatchTouchEvent' || method === 'DOM.getBoxModel')
            console.log(method, JSON.stringify(args), JSON.stringify(result));
          return result;
        } catch (error) {
          console.error('CDP', method, error);
          throw error;
        }
      };
      let callback, armed;
      const ready = new Promise((r) => (armed = r));
      installNativeTrackpad(inputContents, inputContents, () => true, {
        platform: 'darwin',
        load: () => ({
          position: () => ({ x: 270, y: 330 }),
          start: (_handle, cb) => {
            console.log('capture armed');
            callback = cb;
            armed();
            return 1;
          },
          stop: () => {},
        }),
        window: () => inputWindow,
      });
      await win.loadURL(`http://localhost:${server.address().port}/outer`);
      await wc.capturePage(); // Wait for the nested frame's first compositor frame.
      if (!headless) {
        app.focus({ steal: true });
        win.show();
        win.focus();
        wc.focus();
      }
      for (let i = 0; i < 100 && !callback; i++) {
        if (headless) wc.emit('before-mouse-event', {}, { type: 'mouseMove', x: 270, y: 330 });
        else {
          win.focus();
          wc.focus();
          wc.sendInputEvent({ type: 'mouseMove', x: 270, y: 330 });
        }
        await new Promise((r) => setTimeout(r, 10));
      }
      if (!callback)
        console.log(
          'ARM_FAILURE',
          JSON.stringify({
            windowFocus: win.isFocused(),
            guestFocus: wc.isFocused(),
            visible: win.isVisible(),
            pointerEvents,
            commands,
            optIn: await wc.executeJavaScript(
              'document.querySelector("meta[name=cindy-native-trackpad]")?.content',
            ),
          }),
        );
      assert(callback, 'Native surface not armed after focus/DOM readiness');
      await ready;
      const frame = wc.mainFrame.frames[0].frames[0];
      const points = [
        { id: 0, x: 0.3, y: 0.5 },
        { id: 1, x: 0.7, y: 0.5 },
      ];
      const emit = (phase, touches) => {
        console.log('emit', phase);
        callback(JSON.stringify({ phase, touches, width: 100, height: 70 }));
      };
      const until = async (predicate) => {
        for (let i = 0; i < 100; i++) {
          const events = await frame.executeJavaScript('events');
          if (predicate(events)) return events;
          await new Promise((r) => setTimeout(r, 10));
        }
        throw Error(`Touch delivery did not settle; focused=${wc.isFocused()}/${win.isFocused()}`);
      };
      emit('begin', points);
      await until((e) => e.some((v) => v.type === 'touchstart' && v.touches.length === 2));
      emit('change', [points[0], { id: 1, x: 0.8, y: 0.6 }]);
      const events = await until((e) => e.some((v) => v.type === 'touchmove'));
      const start = events.find((v) => v.touches.length === 2).touches;
      const moved = events.findLast((v) => v.type === 'touchmove').touches;
      assert.deepEqual(moved[0], start[0]);
      assert.equal(Math.round(moved[1].x - start[1].x), 50);
      assert.equal(Math.round(moved[1].y - start[1].y), 35);
      // Runtime tier: measure sustained input, not just eventual correctness.
      // Each position identifies its sample across the real renderer boundary.
      const sent = new Map();
      for (let i = 0; i < 120; i++) {
        const x = 0.55 + i / 1000;
        sent.set(Math.round((start[1].x + (x - 0.7) * 500) * 10), Date.now());
        emit('change', [points[0], { id: 1, x, y: 0.5 }]);
        await new Promise((r) => setTimeout(r, 8));
      }
      const measured = await until((e) =>
        e.some(
          (v) => v.type === 'touchmove' && Math.abs(v.touches[1]?.x - (start[1].x - 15.5)) < 0.1,
        ),
      );
      const latency = measured
        .filter((v) => v.type === 'touchmove' && v.touches.length === 2)
        .map((v) =>
          sent.has(Math.round(v.touches[1].x * 10))
            ? v.at - sent.get(Math.round(v.touches[1].x * 10))
            : null,
        )
        .filter((v) => v !== null)
        .sort((a, b) => a - b);
      console.log(
        'INPUT_METRICS',
        JSON.stringify({
          sent: 120,
          received: latency.length,
          p50: latency[Math.floor(latency.length * 0.5)],
          p95: latency[Math.floor(latency.length * 0.95)],
          max: latency.at(-1),
          initialContacts: measured
            .filter((v) => v.type === 'touchstart')
            .map((v) => v.touches.length),
        }),
      );
      if (process.env.CINDY_TRACKPAD_VERIFY_CONTACTS === '1') {
        assert.deepEqual(
          measured.filter((v) => v.type === 'touchstart').map((v) => v.touches.length),
          [2],
        );
        assert.equal(latency.length, 120);
      }
      emit('change', [points[0]]);
      const lifted = await until((e) =>
        e.some((v) => v.type === 'touchend' && v.touches.length === 1),
      );
      assert.equal(lifted.findLast((v) => v.type === 'touchend').touches[0].id, start[0].id);
      emit('end', []);
      await until((e) => e.some((v) => v.type === 'touchend' && v.touches.length === 0));
      if (process.env.CINDY_TRACKPAD_VERIFY_CONTACTS === '1') {
        // React can replace a surface without navigation or pointer movement.
        // The old delivery must cancel, release its object and re-hit-test.
        const previous = callback;
        await frame.executeJavaScript('replaceSurface(false)');
        emit('begin', points);
        for (let i = 0; i < 100 && callback === previous; i++)
          await new Promise((r) => setTimeout(r, 10));
        assert.notEqual(callback, previous, 'Disconnected contacts surface did not rearm');
      }
      emit('begin', points);
      await until(
        (e) => e.filter((v) => v.type === 'touchstart' && v.touches.length === 2).length === 2,
      );
      if (process.env.CINDY_TRACKPAD_VERIFY_CONTACTS === '1') {
        let previous = callback;
        emit('end', []);
        await until((e) => e.at(-1)?.type === 'touchend' && e.at(-1).touches.length === 0);
        await frame.executeJavaScript('replaceSurface(true)');
        emit('begin', points);
        for (let i = 0; i < 100 && callback === previous; i++)
          await new Promise((r) => setTimeout(r, 10));
        assert.notEqual(callback, previous, 'Covered contacts surface did not rearm');
        emit('begin', points);
        await until((e) => e.at(-1)?.type === 'touchstart' && e.at(-1).target === '2');
        previous = callback;
        await frame.executeJavaScript('replaceSurface(false)');
        emit('change', points);
        for (let i = 0; i < 100 && callback === previous; i++)
          await new Promise((r) => setTimeout(r, 10));
        assert.notEqual(callback, previous, 'Mid-gesture replacement did not rearm');
        emit('begin', points);
        const rebound = await until(
          (e) => e.at(-1)?.type === 'touchstart' && e.at(-1).target === '3',
        );
        assert(rebound.some((e) => e.type === 'touchcancel' && e.connected === false));
        console.log(
          'PASS: disconnected, covered and mid-gesture replaced contacts rearm without pointer movement',
        );
        emit('end', []);
        await until((e) => e.at(-1)?.type === 'touchend' && e.at(-1).touches.length === 0);
        await frame.executeJavaScript(`(()=>{
          const canvas=[...document.querySelectorAll('canvas')].at(-1),host=document.createElement('div');
          host.style.display='contents';canvas.style.cssText='position:absolute;left:20px;top:10px;width:400px;height:500px;touch-action:none';
          canvas.replaceWith(host);host.attachShadow({mode:'closed'}).append(canvas);
        })()`);
        previous = callback;
        emit('begin', points);
        await until((e) => e.at(-1)?.type === 'touchstart' && e.at(-1).target === '3');
        assert.equal(callback, previous, 'A valid shadow-root surface must not be rejected');
        console.log('PASS: begin hit testing preserves a valid closed-shadow-root surface');
      }
      win.emit('blur');
      await until((e) => e.at(-1)?.type === 'touchcancel');
      for (let i = 0; i < 100 && wc.debugger.isAttached(); i++)
        await new Promise((r) => setTimeout(r, 10));
      assert.equal(
        wc.debugger.isAttached(),
        false,
        'Cleanup must release its object before detaching',
      );
      console.log(
        'PASS: native addon loads; nested iframe mapping, stationary finger, independent motion, liftoff and blur cancellation',
      );
      win.destroy();
      server.close();
      clearTimeout(deadline);
      app.exit(0);
    } catch (error) {
      console.error(error);
      win?.destroy();
      server?.close();
      clearTimeout(deadline);
      app.exit(1);
    }
  })();
}