Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

12.8 KB
Raw Blame
import { z } from 'zod';
import sharp from 'sharp';
import {
  INLINE_IMAGE_TARGET_BYTES,
  type AndroidDeviceStateResult,
  type AndroidConnectedDevice,
  type AndroidUiNode,
  type AndroidMcpCallContext,
  type AndroidMcpDeps,
  type AndroidMcpErrorCode,
  type AndroidMcpToolName,
  type AndroidStatusSummary,
} from '../types.js';
import {
  androidBusinessError,
  androidTextResult,
  type AndroidToolRegistry,
} from './tool-registry.js';

const CATEGORY = 'android' as const;
const MAX_STATE_TEXT_BYTES = INLINE_IMAGE_TARGET_BYTES;
const MAX_TOOL_RESULT_STREAM_BYTES = 240_000;
const TOOL_RESULT_ENVELOPE_BYTES = 4_096;
const MAX_NODE_STRING_CHARS = 160;
const INLINE_SCREENSHOT_SIDES = [1280, 1024, 800, 640, 480, 360, 240] as const;
const INLINE_SCREENSHOT_QUALITIES = [72, 60, 48, 36, 28, 20] as const;
type InlineScreenshot = {
  data: string;
  mimeType: AndroidDeviceStateResult['screenshot_mime_type'] | 'image/jpeg';
};
type AndroidStatePayload = {
  ok: true;
  data: {
    device_serial: string;
    screen: AndroidDeviceStateResult['screen'];
    current_app: AndroidDeviceStateResult['current_app'];
    screenshot_file_path: string;
    nodes: AndroidUiNode[];
    nodes_truncated?: true;
    raw_ui_dump_file_path?: string;
    ui_dump_error?: string;
  };
};

const DEVICE_SERIAL = z.string().min(1).optional().describe('可选 device serial。多设备连接时必须显式传。');

function isBusinessError(result: unknown): result is {
  ok: false;
  errorCode: AndroidMcpErrorCode;
  message?: string;
  data?: Record<string, unknown>;
} {
  return !!result && typeof result === 'object' && (result as { ok?: unknown }).ok === false
    && typeof (result as { errorCode?: unknown }).errorCode === 'string';
}

function coerceSuccess<T>(tool: AndroidMcpToolName, result: unknown): T {
  if (!result || typeof result !== 'object') {
    throw new Error(`android:${tool}: invalid host result`);
  }
  const value = result as { ok?: unknown; data?: unknown };
  if (value.ok !== true) {
    throw new Error(`android:${tool}: host result was not ok`);
  }
  return value.data as T;
}

async function callAndroidTool<T>(
  deps: AndroidMcpDeps,
  name: AndroidMcpToolName,
  args: Record<string, unknown>,
  context?: AndroidMcpCallContext,
): Promise<T> {
  const result = await deps.callTool(name, args, context);
  if (isBusinessError(result)) {
    throw result;
  }
  return coerceSuccess<T>(name, result);
}

function toErrorResult(err: unknown) {
  if (isBusinessError(err)) {
    return androidBusinessError(
      err.errorCode,
      err.message ?? 'Android MCP host call failed',
      err.data,
    );
  }
  return androidBusinessError(
    'ANDROID_DRIVER_ERROR',
    err instanceof Error ? err.message : String(err),
  );
}

async function compressInlineScreenshot(data: AndroidDeviceStateResult): Promise<InlineScreenshot> {
  const original = Buffer.from(data.screenshot_base64, 'base64');
  if (original.byteLength <= INLINE_IMAGE_TARGET_BYTES) {
    return { data: data.screenshot_base64, mimeType: data.screenshot_mime_type };
  }

  let smallest: Buffer | null = null;
  for (const side of INLINE_SCREENSHOT_SIDES) {
    for (const quality of INLINE_SCREENSHOT_QUALITIES) {
      const resized = await sharp(original, { failOn: 'none' })
        .rotate()
        .resize({
          width: side,
          height: side,
          fit: 'inside',
          withoutEnlargement: true,
        })
        .jpeg({ quality })
        .toBuffer();
      if (!smallest || resized.byteLength < smallest.byteLength) {
        smallest = resized;
      }
      if (resized.byteLength <= INLINE_IMAGE_TARGET_BYTES) {
        return { data: resized.toString('base64'), mimeType: 'image/jpeg' };
      }
    }
  }

  if (smallest) {
    return { data: smallest.toString('base64'), mimeType: 'image/jpeg' };
  }
  return { data: data.screenshot_base64, mimeType: data.screenshot_mime_type };
}

function truncateNodeString(value: string | undefined): {
  value: string | undefined;
  truncated: boolean;
} {
  if (!value || value.length <= MAX_NODE_STRING_CHARS) {
    return { value, truncated: false };
  }
  return {
    value: `${value.slice(0, MAX_NODE_STRING_CHARS)}...`,
    truncated: true,
  };
}

function compactUiNode(node: AndroidUiNode): { node: AndroidUiNode; truncated: boolean } {
  const text = truncateNodeString(node.text);
  const contentDesc = truncateNodeString(node.content_desc);
  const resourceId = truncateNodeString(node.resource_id);
  const className = truncateNodeString(node.class_name);
  const packageName = truncateNodeString(node.package);
  return {
    node: {
      ...node,
      ...(text.value !== undefined ? { text: text.value } : {}),
      ...(contentDesc.value !== undefined ? { content_desc: contentDesc.value } : {}),
      ...(resourceId.value !== undefined ? { resource_id: resourceId.value } : {}),
      ...(className.value !== undefined ? { class_name: className.value } : {}),
      ...(packageName.value !== undefined ? { package: packageName.value } : {}),
    },
    truncated: text.truncated
      || contentDesc.truncated
      || resourceId.truncated
      || className.truncated
      || packageName.truncated,
  };
}

function buildStatePayload(
  data: AndroidDeviceStateResult,
  nodes: AndroidUiNode[],
  nodesTruncated: boolean,
): AndroidStatePayload {
  return {
    ok: true,
    data: {
      device_serial: data.device_serial,
      screen: data.screen,
      current_app: data.current_app,
      screenshot_file_path: data.screenshot_file_path,
      nodes,
      ...(nodesTruncated ? { nodes_truncated: true } : {}),
      ...(data.raw_ui_dump_file_path ? { raw_ui_dump_file_path: data.raw_ui_dump_file_path } : {}),
      ...(data.ui_dump_error ? { ui_dump_error: data.ui_dump_error } : {}),
    },
  };
}

function payloadBytes(payload: AndroidStatePayload): number {
  return Buffer.byteLength(JSON.stringify(payload), 'utf8');
}

function buildBoundedStatePayload(
  data: AndroidDeviceStateResult,
  maxTextBytes = MAX_STATE_TEXT_BYTES,
): AndroidStatePayload {
  const compacted = data.nodes.map(compactUiNode);
  const compactNodes = compacted.map((item) => item.node);
  const stringTruncated = compacted.some((item) => item.truncated);
  const baseTruncated = Boolean(data.nodes_truncated || stringTruncated);
  const fullPayload = buildStatePayload(data, compactNodes, baseTruncated);
  if (payloadBytes(fullPayload) <= maxTextBytes) {
    return fullPayload;
  }

  let low = 0;
  let high = compactNodes.length;
  let best = 0;
  while (low <= high) {
    const mid = Math.floor((low + high) / 2);
    const candidate = buildStatePayload(data, compactNodes.slice(0, mid), true);
    if (payloadBytes(candidate) <= maxTextBytes) {
      best = mid;
      low = mid + 1;
    } else {
      high = mid - 1;
    }
  }
  return buildStatePayload(data, compactNodes.slice(0, best), true);
}

function stateTextBudgetFor(inlineScreenshot: InlineScreenshot): number {
  const imageBlockBytes = Buffer.byteLength(inlineScreenshot.data, 'utf8')
    + Buffer.byteLength(inlineScreenshot.mimeType, 'utf8');
  return Math.max(
    0,
    Math.min(
      MAX_STATE_TEXT_BYTES,
      MAX_TOOL_RESULT_STREAM_BYTES - TOOL_RESULT_ENVELOPE_BYTES - imageBlockBytes,
    ),
  );
}

async function stateResult(data: AndroidDeviceStateResult) {
  let inlineScreenshot: InlineScreenshot = {
    data: data.screenshot_base64,
    mimeType: data.screenshot_mime_type,
  };
  try {
    inlineScreenshot = await compressInlineScreenshot(data);
  } catch {
    inlineScreenshot = { data: data.screenshot_base64, mimeType: data.screenshot_mime_type };
  }

  return {
    content: [
      {
        type: 'text' as const,
        text: JSON.stringify(buildBoundedStatePayload(data, stateTextBudgetFor(inlineScreenshot))),
      },
      {
        type: 'image' as const,
        data: inlineScreenshot.data,
        mimeType: inlineScreenshot.mimeType,
      },
    ],
  };
}

const pointSchema = z.object({
  x: z.number().int().nonnegative(),
  y: z.number().int().nonnegative(),
});

export function registerAndroidTools(
  registry: AndroidToolRegistry,
  deps: AndroidMcpDeps,
  getContext?: () => AndroidMcpCallContext | undefined,
): void {
  registry.register({
    name: 'status',
    category: CATEGORY,
    description: 'Check whether adb is installed and summarize current Android device availability.',
    readOnly: true,
    inputShape: {},
    handler: async () => {
      try {
        const data = await callAndroidTool<AndroidStatusSummary>(deps, 'status', {}, getContext?.());
        return androidTextResult({ ok: true, data });
      } catch (err) {
        return toErrorResult(err);
      }
    },
  });

  registry.register({
    name: 'list_devices',
    category: CATEGORY,
    description: 'List all adb-visible Android devices and emulators with their current states.',
    readOnly: true,
    inputShape: {},
    handler: async () => {
      try {
        const data = await callAndroidTool<AndroidConnectedDevice[]>(deps, 'list_devices', {}, getContext?.());
        return androidTextResult({ ok: true, data });
      } catch (err) {
        return toErrorResult(err);
      }
    },
  });

  registry.register({
    name: 'get_device_state',
    category: CATEGORY,
    description: 'Capture a screenshot, current app, screen size, and compact UI node list for one Android device.',
    readOnly: true,
    inputShape: {
      device_serial: DEVICE_SERIAL,
    },
    handler: async (args) => {
      try {
        const data = await callAndroidTool<AndroidDeviceStateResult>(
          deps,
          'get_device_state',
          args as Record<string, unknown>,
          getContext?.(),
        );
        return await stateResult(data);
      } catch (err) {
        return toErrorResult(err);
      }
    },
  });

  registry.register({
    name: 'tap',
    category: CATEGORY,
    description: 'Tap by element_index from the latest device snapshot or by absolute screen coordinates.',
    inputShape: {
      device_serial: DEVICE_SERIAL,
      element_index: z.number().int().nonnegative().optional(),
      x: z.number().int().nonnegative().optional(),
      y: z.number().int().nonnegative().optional(),
    },
    handler: async (args) => {
      try {
        const data = await callAndroidTool<Record<string, unknown>>(deps, 'tap', args as Record<string, unknown>, getContext?.());
        return androidTextResult({ ok: true, data });
      } catch (err) {
        return toErrorResult(err);
      }
    },
  });

  registry.register({
    name: 'swipe',
    category: CATEGORY,
    description: 'Swipe between absolute screen coordinates on one Android device.',
    inputShape: {
      device_serial: DEVICE_SERIAL,
      start: pointSchema,
      end: pointSchema,
      duration_ms: z.number().int().min(0).max(60_000).optional(),
    },
    handler: async (args) => {
      try {
        const data = await callAndroidTool<Record<string, unknown>>(deps, 'swipe', args as Record<string, unknown>, getContext?.());
        return androidTextResult({ ok: true, data });
      } catch (err) {
        return toErrorResult(err);
      }
    },
  });

  registry.register({
    name: 'input_text',
    category: CATEGORY,
    description: 'Input plain text into the currently focused Android field using adb shell input text.',
    inputShape: {
      device_serial: DEVICE_SERIAL,
      text: z.string().min(1),
    },
    handler: async (args) => {
      try {
        const data = await callAndroidTool<Record<string, unknown>>(deps, 'input_text', args as Record<string, unknown>, getContext?.());
        return androidTextResult({ ok: true, data });
      } catch (err) {
        return toErrorResult(err);
      }
    },
  });

  registry.register({
    name: 'press_key',
    category: CATEGORY,
    description: 'Send one Android keyevent such as BACK, HOME, ENTER, or APP_SWITCH.',
    inputShape: {
      device_serial: DEVICE_SERIAL,
      key: z.enum(['BACK', 'HOME', 'ENTER', 'APP_SWITCH', 'POWER', 'DPAD_UP', 'DPAD_DOWN', 'DPAD_LEFT', 'DPAD_RIGHT', 'DPAD_CENTER']),
    },
    handler: async (args) => {
      try {
        const data = await callAndroidTool<Record<string, unknown>>(deps, 'press_key', args as Record<string, unknown>, getContext?.());
        return androidTextResult({ ok: true, data });
      } catch (err) {
        return toErrorResult(err);
      }
    },
  });

  registry.register({
    name: 'launch_app',
    category: CATEGORY,
    description: 'Launch an Android app by package name and optional activity. Arbitrary intents are not supported.',
    inputShape: {
      device_serial: DEVICE_SERIAL,
      package: z.string().min(1),
      activity: z.string().min(1).optional(),
    },
    handler: async (args) => {
      try {
        const data = await callAndroidTool<Record<string, unknown>>(deps, 'launch_app', args as Record<string, unknown>, getContext?.());
        return androidTextResult({ ok: true, data });
      } catch (err) {
        return toErrorResult(err);
      }
    },
  });
}