High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat: complete OpenClaw plugin with tools, tests, and CI/CD
Luigi Agosti4b2fa0cdd6c4
diff --git a/.github/workflows/ci-plugin.yml b/.github/workflows/ci-plugin.ymlnew file mode 100644index 00000000..40a344fd--- /dev/null+++ b/.github/workflows/ci-plugin.yml@@ -0,0 +1,21 @@+name: CI / Plugin++on:+ push:+ branches: [main]+ paths:+ - 'plugin/**'+ - '.github/workflows/*plugin*'+ pull_request:+ paths:+ - 'plugin/**'+ - '.github/workflows/*plugin*'++permissions:+ contents: read++jobs:+ plugin:+ uses: ./.github/workflows/reusable-plugin.yml+ with:+ ref: ${{ github.sha }}diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex ab16b6cf..c5ac9a51 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -63,6 +63,12 @@ jobs:ref: ${{ needs.validate.outputs.sha }}version: ${{ needs.validate.outputs.version }}+ plugin-checks:+ needs: validate+ uses: ./.github/workflows/reusable-plugin.yml+ with:+ ref: ${{ needs.validate.outputs.sha }}+e2e-api-extended:needs: validateuses: ./.github/workflows/reusable-e2e.yml@@ -157,6 +163,7 @@ jobs:- dashboard-checks- docs-checks- npm-checks+ - plugin-checks- e2e-api-extended- e2e-cli-extended- e2e-infra-extended@@ -170,6 +177,7 @@ jobs:&& needs.dashboard-checks.result == 'success'&& needs.docs-checks.result == 'success'&& needs.npm-checks.result == 'success'+ && needs.plugin-checks.result == 'success'&& needs.publish-dry-run.result == 'success'runs-on: ubuntu-latestenvironment:@@ -186,6 +194,7 @@ jobs:echo "| Dashboard | ✅ |" >> "$GITHUB_STEP_SUMMARY"echo "| Docs | ✅ |" >> "$GITHUB_STEP_SUMMARY"echo "| npm | ✅ |" >> "$GITHUB_STEP_SUMMARY"+ echo "| Plugin | ✅ |" >> "$GITHUB_STEP_SUMMARY"echo "| Publish dry run | ✅ |" >> "$GITHUB_STEP_SUMMARY"echo "| Docker smoke | ${{ needs.docker-smoke.result == 'success' && '✅' || '⚠️ ' }}${{ needs.docker-smoke.result }} |" >> "$GITHUB_STEP_SUMMARY"echo "| E2E API | ${{ needs.e2e-api-extended.result == 'success' && '✅' || '⚠️ ' }}${{ needs.e2e-api-extended.result }} |" >> "$GITHUB_STEP_SUMMARY"diff --git a/.github/workflows/reusable-plugin.yml b/.github/workflows/reusable-plugin.ymlnew file mode 100644index 00000000..6f7038de--- /dev/null+++ b/.github/workflows/reusable-plugin.yml@@ -0,0 +1,54 @@+name: Reusable / Plugin++on:+ workflow_call:+ inputs:+ ref:+ required: true+ type: string+++permissions:+ contents: read++jobs:+ check:+ name: Check Plugin+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v6+ with:+ ref: ${{ inputs.ref }}++ - uses: actions/setup-node@v6+ with:+ node-version: '22'++ - name: Verify JSON schemas+ working-directory: plugin+ run: |+ node -e "JSON.parse(require('fs').readFileSync('package.json', 'utf8'))"+ node -e "JSON.parse(require('fs').readFileSync('openclaw.plugin.json', 'utf8'))"+ echo "✅ All JSON files are valid"++ - name: Verify package contents+ working-directory: plugin+ run: |+ echo "📦 Package will contain:"+ npm pack --dry-run 2>&1 | grep -E "^npm notice [0-9]" | awk '{print " " $4 " (" $3 ")"}'++ test:+ name: Test Plugin+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v6+ with:+ ref: ${{ inputs.ref }}++ - uses: actions/setup-node@v6+ with:+ node-version: '22'++ - name: Run tests+ working-directory: plugin+ run: npx tsx --test *.test.tsdiff --git a/.github/workflows/reusable-publish-plugin.yml b/.github/workflows/reusable-publish-plugin.ymlnew file mode 100644index 00000000..38da9b55--- /dev/null+++ b/.github/workflows/reusable-publish-plugin.yml@@ -0,0 +1,82 @@+name: Reusable / Publish Plugin++on:+ workflow_call:+ inputs:+ version:+ required: true+ type: string+ secrets:+ NPM_TOKEN:+ required: true+ workflow_dispatch:+ inputs:+ version:+ description: 'Plugin version (e.g. 0.3.0, without v prefix)'+ required: true+++permissions:+ contents: read++jobs:+ publish:+ name: Publish to npm+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v6++ - uses: actions/setup-node@v6+ with:+ node-version: '22'+ registry-url: 'https://registry.npmjs.org'++ - name: Resolve version+ id: version+ env:+ INPUT_VERSION: ${{ inputs.version }}+ DISPATCH_VERSION: ${{ github.event.inputs.version }}+ run: |+ if [ -n "$INPUT_VERSION" ]; then+ VERSION="$INPUT_VERSION"+ elif [ -n "$DISPATCH_VERSION" ]; then+ VERSION="$DISPATCH_VERSION"+ else+ VERSION=${GITHUB_REF_NAME#v}+ fi+ VERSION=${VERSION#v}+ echo "value=$VERSION" >> "$GITHUB_OUTPUT"++ - name: Update plugin version+ working-directory: plugin+ env:+ VERSION: ${{ steps.version.outputs.value }}+ run: |+ CURRENT=$(node -pe "require('./package.json').version")+ if [ "$CURRENT" != "$VERSION" ]; then+ npm version "$VERSION" --no-git-tag-version+ # Also update openclaw.plugin.json version+ node -e "+ const fs = require('fs');+ const p = JSON.parse(fs.readFileSync('openclaw.plugin.json', 'utf8'));+ p.version = '$VERSION';+ fs.writeFileSync('openclaw.plugin.json', JSON.stringify(p, null, 2) + '\n');+ "+ else+ echo "Version already correct: $CURRENT"+ fi++ - name: Publish to npm+ working-directory: plugin+ run: npm publish --access public+ env:+ NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}++ - name: Summary+ env:+ VERSION: ${{ steps.version.outputs.value }}+ run: |+ echo "## Plugin Published" >> "$GITHUB_STEP_SUMMARY"+ echo "- Package: @pinchtab/pinchtab" >> "$GITHUB_STEP_SUMMARY"+ echo "- Version: $VERSION" >> "$GITHUB_STEP_SUMMARY"+ echo "- Install: \`openclaw plugins install @pinchtab/pinchtab\`" >> "$GITHUB_STEP_SUMMARY"diff --git a/.github/workflows/reusable-release-publish.yml b/.github/workflows/reusable-release-publish.ymlindex ca0ee094..89a89d85 100644--- a/.github/workflows/reusable-release-publish.yml+++ b/.github/workflows/reusable-release-publish.yml@@ -166,3 +166,14 @@ jobs:version: ${{ inputs.tag }}secrets:CLAWHUB_TOKEN: ${{ secrets.CLAWHUB_TOKEN }}++ plugin:+ name: Publish Plugin+ needs:+ - release+ - npm+ uses: ./.github/workflows/reusable-publish-plugin.yml+ with:+ version: ${{ inputs.tag }}+ secrets:+ NPM_TOKEN: ${{ secrets.NPM_TOKEN }}diff --git a/dev b/devindex 97d024ca..f74a436a 100755--- a/dev+++ b/dev@@ -24,6 +24,7 @@ COMMANDS=("check:🧪:All checks""test unit:🔬:Go unit tests""test dashboard:🔬:Dashboard unit tests"+ "test plugin:🔬:Plugin unit tests""e2e:🐳:E2E suites"" pr:🐳:PR suite (api + cli + infra)"" release:🐳:Release suite (all extended)"@@ -31,6 +32,7 @@ COMMANDS=(" api:🐳:API basic tests"" cli:🐳:CLI basic tests"" infra:🐳:Infra basic tests"+ " plugin:🐳:Plugin smoke tests"" api-extended:🐳:API extended (multi-instance)"" cli-extended:🐳:CLI extended tests"" infra-extended:🐳:Infra extended (multi-instance)"@@ -43,6 +45,7 @@ COMMANDS=("check go:🧪:Go only""check dashboard:🧪:Dashboard only""check npm:🧪:npm package only"+ "check plugin:🧪:Plugin only""check security:🧪:Gosec security scan""check docs:🧪:Validate docs JSON""binary:📦:Build release-style binary into dist/"@@ -77,7 +80,7 @@ run_command() {case "$target" in"check")- echo " ${ACCENT}${BOLD}🧪 Running all checks (Go + Dashboard)${NC}"+ echo " ${ACCENT}${BOLD}🧪 Running all checks (Go + Dashboard + Plugin)${NC}"echo ""bash scripts/check.shlocal go_exit=$?@@ -86,7 +89,12 @@ run_command() {echo ""bash scripts/check-dashboard.shlocal dash_exit=$?- if [ $go_exit -ne 0 ] || [ $dash_exit -ne 0 ]; then exit 1; fi+ echo ""+ echo " ${ACCENT}${BOLD}🧪 Plugin checks${NC}"+ echo ""+ bash scripts/check-plugin.sh+ local plugin_exit=$?+ if [ $go_exit -ne 0 ] || [ $dash_exit -ne 0 ] || [ $plugin_exit -ne 0 ]; then exit 1; fiexit 0;;"check go")@@ -104,6 +112,11 @@ run_command() {echo ""exec bash scripts/check-npm.sh;;+ "check plugin")+ echo " ${ACCENT}${BOLD}🧪 Plugin checks${NC}"+ echo ""+ exec bash scripts/check-plugin.sh+ ;;"check security")echo " ${ACCENT}${BOLD}🔒 Security scan${NC}"echo ""@@ -134,6 +147,11 @@ run_command() {echo ""exec bash scripts/test.sh dashboard;;+ "test plugin")+ echo " ${ACCENT}${BOLD}🔬 Plugin tests${NC}"+ echo ""+ cd plugin && exec npx tsx --test *.test.ts+ ;;"unit")echo " ${ACCENT}${BOLD}🔬 Unit tests${NC}"echo ""diff --git a/plugin/README.md b/plugin/README.mdindex 54cace7d..312a349e 100644--- a/plugin/README.md+++ b/plugin/README.md@@ -9,19 +9,17 @@ openclaw plugins install @pinchtab/pinchtabopenclaw gateway restart```-## Prerequisites+## Quick Start-```bash-# Start Pinchtab-pinchtab &--# With auth token (recommended)-PINCHTAB_TOKEN=my-secret pinchtab &+The plugin auto-starts a local Pinchtab server when needed. Just install and go:-# Docker-docker run -d -p 9867:9867 ghcr.io/pinchtab/pinchtab:latest+```bash+openclaw plugins install @pinchtab/pinchtab+openclaw gateway restart```+For remote servers or Docker, disable auto-start and set `baseUrl`.+## Configure```json5@@ -31,9 +29,38 @@ docker run -d -p 9867:9867 ghcr.io/pinchtab/pinchtab:latestpinchtab: {enabled: true,config: {+ // ConnectionbaseUrl: "http://localhost:9867",token: "my-secret",- timeout: 30000,+ timeoutMs: 30000,++ // Startup (local only)+ autoStart: true, // auto-start server if not running+ binaryPath: "pinchtab", // path to binary+ startupTimeoutMs: 30000, // max wait for startup++ // Policy+ allowEvaluate: false, // block JS evaluate by default+ allowedDomains: [], // empty = allow all+ allowDownloads: false,+ allowUploads: false,++ // Defaults+ defaultSnapshotFormat: "compact",+ defaultSnapshotFilter: "interactive",+ screenshotFormat: "jpeg",+ screenshotQuality: 80,++ // Session+ persistSessionTabs: true, // remember last active tab++ // Tools & Profiles+ registerBrowserTool: true, // register OpenClaw-compatible 'browser' tool+ defaultProfile: "openclaw",+ profiles: {+ "staging": { instanceId: "staging-instance" },+ "user": { attach: true }+ },},},},@@ -47,7 +74,66 @@ docker run -d -p 9867:9867 ghcr.io/pinchtab/pinchtab:latest}```-## Single Tool, All Actions+### Manual Server Setup++If auto-start is disabled or you're using Docker:++```bash+# Local+PINCHTAB_TOKEN=my-secret pinchtab server &++# Docker+docker run -d -p 9867:9867 ghcr.io/pinchtab/pinchtab:latest+```++## Two Tools: `browser` and `pinchtab`++The plugin registers two tools:++| Tool | Use Case |+|------|----------|+| `browser` | OpenClaw-compatible, simplified interface for common flows |+| `pinchtab` | Advanced control with all actions (mouse, wait, handoff, evaluate) |++Disable the browser tool with `registerBrowserTool: false` if you only want `pinchtab`.++## Profiles++Map browser sessions to OpenClaw profile semantics:++| Profile | Behavior |+|---------|----------|+| `openclaw` | Default isolated automation profile |+| `user` | Attach to existing browser session (cookies/logins preserved) |+| Custom | Map to specific Pinchtab instance via config |++```json5+{+ config: {+ defaultProfile: "openclaw",+ profiles: {+ "staging": { instanceId: "staging-browser" },+ "user": { attach: true }+ }+ }+}+```++Usage: `browser({ action: "navigate", url: "...", profile: "user" })`++## Browser Tool Actions++| Action | Description |+|--------|-------------|+| `navigate` | Go to URL (url, profile?, newTab?) |+| `snapshot` | Accessibility tree (selector?, format?, maxTokens?) |+| `screenshot` | Capture image (quality?, format?) |+| `click/type/fill/press/hover/scroll/select` | Element actions (ref, text?, key?) |+| `tabs` | List/new/close tabs (tabAction?, url?, tabId?) |+| `pdf` | Export PDF (landscape?, scale?) |+| `status` | Health check with config/warnings |++## Pinchtab Tool: All ActionsOne tool definition, many actions — keeps context lean:@@ -64,6 +150,9 @@ One tool definition, many actions — keeps context lean:| `screenshot` | JPEG screenshot (vision fallback) | ~2K || `evaluate` | Run JavaScript in page | — || `pdf` | Export page as PDF | — |+| `download` | Download file from URL | — |+| `upload` | Upload files to file input | — |+| `network` | Capture/inspect network requests | — || `health` | Check connectivity | — |## Agent Usage Example@@ -133,11 +222,54 @@ pinchtab({ action: "wait", text: "Welcome back", timeout: 120000 })## Security Notes-- **`evaluate`** executes arbitrary JavaScript in the page — restrict to trusted agents and domains+- **`evaluate`** is blocked by default (`allowEvaluate: false`) — enable only for trusted agents+- Use `allowedDomains` to restrict navigation (e.g., `["*.example.com"]`)- Use `PINCHTAB_TOKEN` to gate API access; rotate regularly- In production, run behind HTTPS reverse proxy (Caddy/nginx)+## Migrating from OpenClaw Bundled Browser++To replace the bundled `browser` plugin with Pinchtab:++### 1. Disable bundled browser+```json5+{+ plugins: {+ deny: ["browser"], // disable bundled+ entries: {+ pinchtab: { enabled: true }+ }+ }+}+```++### 2. Action mapping++| OpenClaw `browser` | Pinchtab equivalent |+|--------------------|---------------------|+| `browser.open(url)` | `browser({ action: "navigate", url })` |+| `browser.snapshot()` | `browser({ action: "snapshot" })` |+| `browser.screenshot()` | `browser({ action: "screenshot" })` |+| `browser.act({ kind: "click", ref })` | `browser({ action: "click", ref })` |+| `browser.act({ kind: "type", ref, text })` | `browser({ action: "type", ref, text })` |+| `browser.tabs()` | `browser({ action: "tabs" })` |+| `browser.status()` | `browser({ action: "status" })` |++### 3. Profile mapping++| OpenClaw profile | Pinchtab config |+|------------------|-----------------|+| `openclaw` (default) | Default isolated profile |+| `user` | `{ attach: true }` - existing session |+| Custom CDP | `profiles: { "name": { instanceId: "..." } }` |++### 4. Key differences++- **Auto-start**: Pinchtab auto-starts locally by default (disable with `autoStart: false`)+- **Policy**: `allowEvaluate`, `allowDownloads`, `allowUploads` are `false` by default+- **Advanced actions**: Use `pinchtab` tool for mouse controls, wait, handoff, evaluate+## Requirements-- Running Pinchtab instance (Go binary or Docker)+- Pinchtab binary in PATH (or set `binaryPath`)- OpenClaw Gatewaydiff --git a/plugin/client.test.ts b/plugin/client.test.tsnew file mode 100644index 00000000..11a2adad--- /dev/null+++ b/plugin/client.test.ts@@ -0,0 +1,120 @@+import { describe, it } from "node:test";+import assert from "node:assert";+import { isRefToken, normalizeActionParams, looksLikeStaleRef, textResult, imageResult, resourceResult } from "./client.ts";++describe("isRefToken", () => {+ it("returns true for valid ref tokens", () => {+ assert.strictEqual(isRefToken("e5"), true);+ assert.strictEqual(isRefToken("e123"), true);+ assert.strictEqual(isRefToken("E5"), true);+ assert.strictEqual(isRefToken("e0"), true);+ });++ it("returns false for invalid ref tokens", () => {+ assert.strictEqual(isRefToken("e"), false);+ assert.strictEqual(isRefToken("5"), false);+ assert.strictEqual(isRefToken("ref5"), false);+ assert.strictEqual(isRefToken(""), false);+ assert.strictEqual(isRefToken(null), false);+ assert.strictEqual(isRefToken(undefined), false);+ assert.strictEqual(isRefToken(123), false);+ });++ it("handles whitespace", () => {+ assert.strictEqual(isRefToken(" e5 "), true);+ assert.strictEqual(isRefToken("e5\n"), true);+ });+});++describe("normalizeActionParams", () => {+ it("converts ref-like selector to ref", () => {+ const result = normalizeActionParams({ selector: "e5" });+ assert.strictEqual(result.ref, "e5");+ assert.strictEqual(result.selector, undefined);+ });++ it("preserves non-ref selector", () => {+ const result = normalizeActionParams({ selector: "#button" });+ assert.strictEqual(result.selector, "#button");+ assert.strictEqual(result.ref, undefined);+ });++ it("does not override existing ref", () => {+ const result = normalizeActionParams({ ref: "e10", selector: "e5" });+ assert.strictEqual(result.ref, "e10");+ assert.strictEqual(result.selector, "e5");+ });++ it("trims ref whitespace", () => {+ const result = normalizeActionParams({ ref: " e5 " });+ assert.strictEqual(result.ref, "e5");+ });++ it("preserves other params", () => {+ const result = normalizeActionParams({ action: "click", ref: "e5", text: "hello" });+ assert.strictEqual(result.action, "click");+ assert.strictEqual(result.ref, "e5");+ assert.strictEqual(result.text, "hello");+ });+});++describe("looksLikeStaleRef", () => {+ it("returns false for non-error results", () => {+ assert.strictEqual(looksLikeStaleRef({ success: true }), false);+ assert.strictEqual(looksLikeStaleRef(null), false);+ assert.strictEqual(looksLikeStaleRef(undefined), false);+ });++ it("returns true for stale ref errors", () => {+ assert.strictEqual(looksLikeStaleRef({ error: "stale element reference" }), true);+ assert.strictEqual(looksLikeStaleRef({ error: "ref not found" }), true);+ assert.strictEqual(looksLikeStaleRef({ error: "no node with id" }), true);+ assert.strictEqual(looksLikeStaleRef({ error: "unknown element" }), true);+ assert.strictEqual(looksLikeStaleRef({ error: "context canceled" }), true);+ });++ it("returns false for other errors", () => {+ assert.strictEqual(looksLikeStaleRef({ error: "network timeout" }), false);+ assert.strictEqual(looksLikeStaleRef({ error: "server error" }), false);+ });++ it("checks body as well as error", () => {+ assert.strictEqual(looksLikeStaleRef({ error: "failed", body: "stale ref" }), true);+ });+});++describe("textResult", () => {+ it("wraps string as text content", () => {+ const result = textResult("hello");+ assert.deepStrictEqual(result, { content: [{ type: "text", text: "hello" }] });+ });++ it("wraps object as JSON text", () => {+ const result = textResult({ key: "value" });+ assert.strictEqual(result.content[0].type, "text");+ assert.ok(result.content[0].text.includes('"key"'));+ });++ it("uses text property if present", () => {+ const result = textResult({ text: "extracted" });+ assert.deepStrictEqual(result, { content: [{ type: "text", text: "extracted" }] });+ });+});++describe("imageResult", () => {+ it("creates image content", () => {+ const result = imageResult("base64data", "image/png");+ assert.deepStrictEqual(result, {+ content: [{ type: "image", data: "base64data", mimeType: "image/png" }],+ });+ });+});++describe("resourceResult", () => {+ it("creates resource content", () => {+ const result = resourceResult("pdf://export", "application/pdf", "base64blob");+ assert.deepStrictEqual(result, {+ content: [{ type: "resource", resource: { uri: "pdf://export", mimeType: "application/pdf", blob: "base64blob" } }],+ });+ });+});diff --git a/plugin/client.ts b/plugin/client.tsnew file mode 100644index 00000000..8f761838--- /dev/null+++ b/plugin/client.ts@@ -0,0 +1,92 @@+import type { PluginConfig, ToolResult } from "./types.js";++export async function pinchtabFetch(+ cfg: PluginConfig,+ path: string,+ opts: { method?: string; body?: unknown; rawResponse?: boolean } = {},+): Promise<any> {+ const base = cfg.baseUrl || "http://localhost:9867";+ const url = `${base}${path}`;+ const headers: Record<string, string> = {};+ if (cfg.token) headers["Authorization"] = `Bearer ${cfg.token}`;+ if (opts.body) headers["Content-Type"] = "application/json";++ const controller = new AbortController();+ const timeout = cfg.timeoutMs ?? cfg.timeout ?? 30000;+ const timer = setTimeout(() => controller.abort(), timeout);++ try {+ const res = await fetch(url, {+ method: opts.method || (opts.body ? "POST" : "GET"),+ headers,+ body: opts.body ? JSON.stringify(opts.body) : undefined,+ signal: controller.signal,+ });+ if (opts.rawResponse) return res;+ const text = await res.text();+ if (!res.ok) {+ return { error: `${res.status} ${res.statusText}`, body: text };+ }+ try {+ return JSON.parse(text);+ } catch {+ return { text };+ }+ } catch (err: any) {+ if (err?.name === "AbortError") {+ return { error: `Request timed out after ${timeout}ms: ${path}` };+ }+ return {+ error: `Connection failed: ${err?.message || err}. Is Pinchtab running at ${base}?`,+ };+ } finally {+ clearTimeout(timer);+ }+}++export function textResult(data: any): ToolResult {+ const text =+ typeof data === "string" ? data : data?.text ?? JSON.stringify(data, null, 2);+ return { content: [{ type: "text", text }] };+}++export function imageResult(b64: string, mimeType: string): ToolResult {+ return { content: [{ type: "image", data: b64, mimeType }] };+}++export function resourceResult(uri: string, mimeType: string, blob: string): ToolResult {+ return { content: [{ type: "resource", resource: { uri, mimeType, blob } }] };+}++export function isRefToken(value: unknown): value is string {+ return typeof value === "string" && /^e\d+$/i.test(value.trim());+}++export function normalizeActionParams(input: any): any {+ const params = { ...input };+ if (!params.ref && isRefToken(params.selector)) {+ params.ref = String(params.selector).trim();+ delete params.selector;+ }+ if (typeof params.ref === "string") {+ params.ref = params.ref.trim();+ }+ return params;+}++export function actionErrorText(result: any): string {+ return `${result?.error || ""} ${result?.body || ""}`.toLowerCase();+}++export function looksLikeStaleRef(result: any): boolean {+ if (!result?.error) return false;+ const text = actionErrorText(result);+ return (+ text.includes("stale") ||+ text.includes("context canceled") ||+ text.includes("ref") ||+ text.includes("not found") ||+ text.includes("no node") ||+ text.includes("unknown element")+ );+}diff --git a/plugin/index.ts b/plugin/index.tsindex d3867f1f..d5b243a3 100644--- a/plugin/index.ts+++ b/plugin/index.ts@@ -1,557 +1,47 @@/*** Pinchtab OpenClaw Plugin*- * Single-tool design: one `pinchtab` tool with an `action` parameter.- * Minimal context bloat — one tool definition covers all browser operations.+ * Two tools:+ * - `pinchtab`: Full-featured browser control with all actions+ * - `browser`: OpenClaw-compatible simplified interface*/-interface PluginConfig {- baseUrl?: string;- token?: string;- timeout?: number;-}--interface PluginApi {- config: { plugins?: { entries?: Record<string, { config?: PluginConfig }> } };- registerTool: (tool: any, opts?: { optional?: boolean }) => void;-}--function isRefToken(value: unknown): value is string {- return typeof value === "string" && /^e\d+$/i.test(value.trim());-}--function normalizeActionParams(input: any): any {- const params = { ...input };- // Some agents send ref-like values (e.g. "e56") in selector.- // Normalize to ref so action routing stays stable.- if (!params.ref && isRefToken(params.selector)) {- params.ref = String(params.selector).trim();- delete params.selector;- }- if (typeof params.ref === "string") {- params.ref = params.ref.trim();- }- return params;-}--function actionErrorText(result: any): string {- return `${result?.error || ""} ${result?.body || ""}`.toLowerCase();-}--function looksLikeStaleRef(result: any): boolean {- if (!result?.error) return false;- const text = actionErrorText(result);- return (- text.includes("stale") ||- text.includes("context canceled") ||- text.includes("ref") ||- text.includes("not found") ||- text.includes("no node") ||- text.includes("unknown element")- );-}+import type { PluginApi, PluginConfig } from "./types.js";+import { pinchtabToolSchema, pinchtabToolDescription, executePinchtabAction } from "./tools/pinchtab.js";+import { browserToolSchema, browserToolDescription, executeBrowserAction } from "./tools/browser.js";function getConfig(api: PluginApi): PluginConfig {return api.config?.plugins?.entries?.pinchtab?.config ?? {};}-async function pinchtabFetch(- cfg: PluginConfig,- path: string,- opts: { method?: string; body?: unknown; rawResponse?: boolean } = {},-): Promise<any> {- const base = cfg.baseUrl || "http://localhost:9867";- const url = `${base}${path}`;- const headers: Record<string, string> = {};- if (cfg.token) headers["Authorization"] = `Bearer ${cfg.token}`;- if (opts.body) headers["Content-Type"] = "application/json";-- const controller = new AbortController();- const timeout = cfg.timeout || 30000;- const timer = setTimeout(() => controller.abort(), timeout);-- try {- const res = await fetch(url, {- method: opts.method || (opts.body ? "POST" : "GET"),- headers,- body: opts.body ? JSON.stringify(opts.body) : undefined,- signal: controller.signal,- });- if (opts.rawResponse) return res;- const text = await res.text();- if (!res.ok) {- return { error: `${res.status} ${res.statusText}`, body: text };- }- try {- return JSON.parse(text);- } catch {- return { text };- }- } catch (err: any) {- if (err?.name === "AbortError") {- return { error: `Request timed out after ${timeout}ms: ${path}` };- }- return {- error: `Connection failed: ${err?.message || err}. Is Pinchtab running at ${base}?`,- };- } finally {- clearTimeout(timer);- }-}--function textResult(data: any): any {- const text =- typeof data === "string" ? data : data?.text ?? JSON.stringify(data, null, 2);- return { content: [{ type: "text", text }] };-}-export default function register(api: PluginApi) {+ const cfg = getConfig(api);++ // Register the full-featured pinchtab toolapi.registerTool({name: "pinchtab",- description: `Browser control via Pinchtab. Actions:-- navigate: go to URL (url, tabId?, newTab?, blockImages?, timeout?)-- snapshot: accessibility tree (filter?, format?, selector?, maxTokens?, depth?, diff?, tabId?)- - click/type/press/fill/hover/scroll/select/focus: act on element (ref, text?, key?, value?, scrollY?, waitNav?, tabId?)- - mouse-move/mouse-down/mouse-up/mouse-wheel: low-level mouse controls (ref|selector|x+y, button?, deltaX?, deltaY?, tabId?)-- text: extract readable text (mode?, tabId?)-- wait: pause until condition (selector|text|url|load|fn|ms, tabId?, timeout?, state?)-- handoff: request human intervention mid-flow (captcha/login/2FA/credentials), optionally wait for resume condition-- tabs: list/new/close tabs (tabAction?, url?, tabId?)-- screenshot: JPEG screenshot (quality?, tabId?)-- evaluate: run JS (expression, tabId?)-- pdf: export page as PDF (landscape?, scale?, tabId?)-- health: check connectivity--Token strategy: use "text" for reading (~800 tokens), "snapshot" with filter=interactive&format=compact for interactions (~3,600), diff=true on subsequent snapshots.`,- parameters: {- type: "object",- properties: {- action: {- type: "string",- enum: [- "navigate",- "snapshot",- "click",- "type",- "press",- "fill",- "hover",- "mouse-move",- "mouse-down",- "mouse-up",- "mouse-wheel",- "scroll",- "select",- "focus",- "text",- "wait",- "handoff",- "tabs",- "screenshot",- "evaluate",- "pdf",- "health",- ],- description: "Action to perform",- },- url: { type: "string", description: "URL for navigate or new tab" },- ref: {- type: "string",- description: "Element ref from snapshot (e.g. e5)",- },- text: { type: "string", description: "Text to type or fill" },- key: {- type: "string",- description: "Key to press (e.g. Enter, Tab, Escape)",- },- expression: {- type: "string",- description: "JavaScript expression for evaluate",- },- selector: {- type: "string",- description: "CSS selector for snapshot scope or action target",- },- filter: {- type: "string",- enum: ["interactive", "all"],- description: "Snapshot filter: interactive = buttons/links/inputs only",- },- format: {- type: "string",- enum: ["json", "compact", "text", "yaml"],- description: "Snapshot format: compact is most token-efficient",- },- maxTokens: {- type: "number",- description: "Truncate snapshot to ~N tokens",- },- depth: { type: "number", description: "Max snapshot tree depth" },- diff: {- type: "boolean",- description: "Snapshot diff: only changes since last snapshot",- },- value: { type: "string", description: "Value for select dropdown" },- scrollY: {- type: "number",- description: "Pixels to scroll vertically",- },- x: {- type: "number",- description: "Mouse X coordinate (used by low-level mouse actions)",- },- y: {- type: "number",- description: "Mouse Y coordinate (used by low-level mouse actions)",- },- button: {- type: "string",- enum: ["left", "right", "middle"],- description: "Mouse button for mouse-down/mouse-up",- },- deltaX: {- type: "number",- description: "Mouse wheel horizontal delta for mouse-wheel",- },- deltaY: {- type: "number",- description: "Mouse wheel vertical delta for mouse-wheel",- },- waitNav: {- type: "boolean",- description: "Wait for navigation after action",- },- tabId: { type: "string", description: "Target tab ID" },- tabAction: {- type: "string",- enum: ["list", "new", "close"],- description: "Tab sub-action (default: list)",- },- newTab: { type: "boolean", description: "Open URL in new tab" },- blockImages: { type: "boolean", description: "Block image loading" },- timeout: {- type: "number",- description: "Navigation timeout in seconds",- },- quality: {- type: "number",- description: "JPEG quality 1-100 (default: 80)",- },- mode: {- type: "string",- enum: ["readability", "raw"],- description: "Text extraction mode",- },- ms: {- type: "number",- description: "Wait milliseconds for wait/handoff actions",- },- state: {- type: "string",- enum: ["visible", "hidden", "attached", "detached"],- description: "Wait state for selector waits",- },- load: {- type: "string",- enum: ["load", "domcontentloaded", "networkidle"],- description: "Document load state for wait action",- },- fn: {- type: "string",- description: "JavaScript predicate string for wait action",- },- humanReason: {- type: "string",- description:- "Reason for manual handoff (captcha/login/2FA/credential input)",- },- humanPrompt: {- type: "string",- description: "Instruction shown when handing off to a human",- },- landscape: { type: "boolean", description: "PDF landscape orientation" },- scale: { type: "number", description: "PDF print scale (default: 1.0)" },- },- required: ["action"],- },+ description: pinchtabToolDescription,+ parameters: pinchtabToolSchema,async execute(_id: string, params: any) {- const cfg = getConfig(api);- const normalized = normalizeActionParams(params);- const { action } = normalized;-- // --- navigate ---- if (action === "navigate") {- const body: any = { url: normalized.url };- if (normalized.tabId) body.tabId = normalized.tabId;- if (normalized.newTab) body.newTab = true;- if (normalized.blockImages) body.blockImages = true;- if (normalized.timeout) body.timeout = normalized.timeout;- return textResult(await pinchtabFetch(cfg, "/navigate", { body }));- }-- // --- snapshot ---- if (action === "snapshot") {- const query = new URLSearchParams();- if (normalized.tabId) query.set("tabId", normalized.tabId);- if (normalized.filter) query.set("filter", normalized.filter);- if (normalized.format) query.set("format", normalized.format);- if (normalized.selector) query.set("selector", normalized.selector);- if (normalized.maxTokens) query.set("maxTokens", String(normalized.maxTokens));- if (normalized.depth) query.set("depth", String(normalized.depth));- if (normalized.diff) query.set("diff", "true");- const qs = query.toString();- return textResult(- await pinchtabFetch(cfg, `/snapshot${qs ? `?${qs}` : ""}`),- );- }-- // --- element actions ---- const elementActions = [- "click",- "type",- "press",- "fill",- "hover",- "mouse-move",- "mouse-down",- "mouse-up",- "mouse-wheel",- "scroll",- "select",- "focus",- ];- if (elementActions.includes(action)) {- const body: any = { kind: action };- for (const k of [- "ref",- "text",- "key",- "selector",- "value",- "scrollY",- "x",- "y",- "button",- "deltaX",- "deltaY",- "tabId",- "waitNav",- ]) {- if (normalized[k] !== undefined) body[k] = normalized[k];- }-- let result = await pinchtabFetch(cfg, "/action", { body });-- // One bounded retry: refresh snapshot once when refs look stale.- if (body.ref && looksLikeStaleRef(result)) {- const q = new URLSearchParams();- q.set("filter", "interactive");- q.set("format", "compact");- if (body.tabId) q.set("tabId", body.tabId);- await pinchtabFetch(cfg, `/snapshot?${q.toString()}`);- const retried = await pinchtabFetch(cfg, "/action", { body });- if (!retried?.error) {- result = {- ...retried,- warning:- "Action succeeded after one automatic snapshot refresh (stale ref recovery).",- };- } else {- result = {- ...retried,- warning:- "Action retried once after snapshot refresh but still failed. Refresh refs and retry.",- };- }- }-- // Controlled input fallback: if fill fails, try type once.- if (- action === "fill" &&- result?.error &&- body.ref &&- (typeof body.text === "string" || typeof body.value === "string")- ) {- const typeBody: any = { ...body, kind: "type" };- if (typeof typeBody.text !== "string" && typeof typeBody.value === "string") {- typeBody.text = typeBody.value;- }- delete typeBody.value;- const typed = await pinchtabFetch(cfg, "/action", { body: typeBody });- if (!typed?.error) {- result = {- ...typed,- warning:- "Fill failed; type fallback succeeded (useful for controlled inputs).",- };- }- }-- return textResult(result);- }-- // --- text ---- if (action === "text") {- const query = new URLSearchParams();- if (normalized.tabId) query.set("tabId", normalized.tabId);- if (normalized.mode) query.set("mode", normalized.mode);- const qs = query.toString();- return textResult(- await pinchtabFetch(cfg, `/text${qs ? `?${qs}` : ""}`),- );- }-- // --- wait ---- if (action === "wait") {- const body: any = {};- for (const k of ["selector", "text", "url", "load", "fn", "ms", "tabId", "timeout", "state"]) {- if (normalized[k] !== undefined) body[k] = normalized[k];- }- return textResult(await pinchtabFetch(cfg, "/wait", { body }));- }-- // --- handoff ---- if (action === "handoff") {- const hasWaitCondition = ["selector", "text", "url", "load", "fn", "ms"].some(- (k) => normalized[k] !== undefined,- );-- const handoffMeta = {- status: "human_handoff_required",- reason:- normalized.humanReason ||- "Manual intervention required (captcha/login/2FA/credential entry).",- instructions:- normalized.humanPrompt ||- "Please complete the step in the headed browser, then resume automation.",- };-- if (!hasWaitCondition) {- return textResult({- ...handoffMeta,- resumed: false,- next:- "Call action='handoff' with a wait condition (selector/text/url/load/fn/ms) or use action='wait' to resume when ready.",- });- }-- const waitBody: any = {};- for (const k of ["selector", "text", "url", "load", "fn", "ms", "tabId", "timeout", "state"]) {- if (normalized[k] !== undefined) waitBody[k] = normalized[k];- }- const waitResult = await pinchtabFetch(cfg, "/wait", { body: waitBody });- return textResult({- ...handoffMeta,- resumed: !waitResult?.error,- waitResult,- });- }-- // --- tabs ---- if (action === "tabs") {- const tabAction = normalized.tabAction || "list";- if (tabAction === "list") {- const listed = await pinchtabFetch(cfg, "/tabs");- const tabs = Array.isArray(listed?.tabs)- ? listed.tabs- : Array.isArray(listed)- ? listed- : [];- if (tabs.length > 0) {- return textResult(listed);- }-- // Fallback: when global /tabs is empty, try the first running instance.- const instances = await pinchtabFetch(cfg, "/instances");- const list = Array.isArray(instances?.value)- ? instances.value- : Array.isArray(instances)- ? instances- : [];- const running = list.find((i: any) => i?.status === "running" && i?.id);- if (!running) {- return textResult({- ...listed,- warning:- "No tabs returned from /tabs and no running instance found for fallback.",- });- }- const instanceTabs = await pinchtabFetch(- cfg,- `/instances/${running.id}/tabs`,- );- return textResult({- source: "instance-fallback",- instanceId: running.id,- tabs: instanceTabs?.tabs ?? instanceTabs,- warning:- "Global /tabs was empty; used /instances/{id}/tabs fallback.",- });- }- const body: any = { action: tabAction };- if (normalized.url) body.url = normalized.url;- if (normalized.tabId) body.tabId = normalized.tabId;- return textResult(await pinchtabFetch(cfg, "/tab", { body }));- }-- // --- screenshot ---- if (action === "screenshot") {- const query = new URLSearchParams();- if (normalized.tabId) query.set("tabId", normalized.tabId);- if (normalized.quality) query.set("quality", String(normalized.quality));- const qs = query.toString();- try {- const res = await pinchtabFetch(- cfg,- `/screenshot${qs ? `?${qs}` : ""}`,- { rawResponse: true },- );- if (res instanceof Response) {- if (!res.ok) {- return textResult({- error: `Screenshot failed: ${res.status} ${await res.text()}`,- });- }- const buf = await res.arrayBuffer();- const b64 = Buffer.from(buf).toString("base64");- return {- content: [{ type: "image", data: b64, mimeType: "image/jpeg" }],- };- }- return textResult(res);- } catch (err: any) {- return textResult({ error: `Screenshot failed: ${err?.message}` });- }- }-- // --- evaluate ---- if (action === "evaluate") {- const body: any = { expression: normalized.expression };- if (normalized.tabId) body.tabId = normalized.tabId;- return textResult(await pinchtabFetch(cfg, "/evaluate", { body }));- }-- // --- pdf ---- if (action === "pdf") {- const query = new URLSearchParams();- if (normalized.tabId) query.set("tabId", normalized.tabId);- if (normalized.landscape) query.set("landscape", "true");- if (normalized.scale) query.set("scale", String(normalized.scale));- const qs = query.toString();- return textResult(- await pinchtabFetch(cfg, `/pdf${qs ? `?${qs}` : ""}`),- );- }-- // --- health ---- if (action === "health") {- return textResult(await pinchtabFetch(cfg, "/health"));- }-- return textResult({ error: `Unknown action: ${action}` });+ return executePinchtabAction(getConfig(api), params);},},{ optional: true },);++ // Register OpenClaw-compatible browser tool+ if (cfg.registerBrowserTool !== false) {+ api.registerTool(+ {+ name: "browser",+ description: browserToolDescription,+ parameters: browserToolSchema,+ async execute(_id: string, params: any) {+ return executeBrowserAction(getConfig(api), params);+ },+ },+ { optional: true },+ );+ }}diff --git a/plugin/openclaw.plugin.json b/plugin/openclaw.plugin.jsonindex ecc4ccec..ac6d7c75 100644--- a/plugin/openclaw.plugin.json+++ b/plugin/openclaw.plugin.json@@ -2,7 +2,7 @@"id": "pinchtab","name": "Pinchtab","description": "Browser control for AI agents via Pinchtab. Single tool, all actions: navigate, snapshot, click/type/press/fill/hover/scroll/select/focus, mouse-move/mouse-down/mouse-up/mouse-wheel, text, screenshot, evaluate, pdf.",- "version": "0.1.0",+ "version": "0.3.0","skills": ["../skills/pinchtab"],"configSchema": {"type": "object",@@ -17,16 +17,114 @@"type": "string","description": "Bearer auth token (PINCHTAB_TOKEN)"},- "timeout": {+ "timeoutMs": {"type": "number","description": "Default request timeout in milliseconds","default": 30000+ },+ "autoStart": {+ "type": "boolean",+ "description": "Auto-start local Pinchtab server if not running",+ "default": true+ },+ "binaryPath": {+ "type": "string",+ "description": "Path to pinchtab binary (default: search PATH)"+ },+ "startupTimeoutMs": {+ "type": "number",+ "description": "Max time to wait for server startup",+ "default": 30000+ },+ "allowEvaluate": {+ "type": "boolean",+ "description": "Allow evaluate action (JS execution)",+ "default": false+ },+ "allowedDomains": {+ "type": "array",+ "items": { "type": "string" },+ "description": "Domain allowlist for navigation (empty = allow all)"+ },+ "allowDownloads": {+ "type": "boolean",+ "description": "Allow file downloads",+ "default": false+ },+ "allowUploads": {+ "type": "boolean",+ "description": "Allow file uploads",+ "default": false+ },+ "defaultSnapshotFormat": {+ "type": "string",+ "enum": ["compact", "json", "text", "yaml"],+ "description": "Default snapshot format",+ "default": "compact"+ },+ "defaultSnapshotFilter": {+ "type": "string",+ "enum": ["interactive", "all"],+ "description": "Default snapshot filter",+ "default": "interactive"+ },+ "screenshotFormat": {+ "type": "string",+ "enum": ["jpeg", "png"],+ "description": "Screenshot image format",+ "default": "jpeg"+ },+ "screenshotQuality": {+ "type": "number",+ "description": "JPEG quality (1-100)",+ "default": 80+ },+ "persistSessionTabs": {+ "type": "boolean",+ "description": "Remember last active tab across calls",+ "default": true+ },+ "registerBrowserTool": {+ "type": "boolean",+ "description": "Register OpenClaw-compatible 'browser' tool",+ "default": true+ },+ "defaultProfile": {+ "type": "string",+ "description": "Default profile name",+ "default": "openclaw"+ },+ "profiles": {+ "type": "object",+ "description": "Profile name to Pinchtab instance/profile mapping",+ "additionalProperties": {+ "type": "object",+ "properties": {+ "instanceId": { "type": "string" },+ "attach": { "type": "boolean" }+ }+ }}}},"uiHints": {"baseUrl": { "label": "Base URL", "placeholder": "http://localhost:9867" },"token": { "label": "Auth Token", "sensitive": true },- "timeout": { "label": "Timeout (ms)", "placeholder": "30000" }+ "timeoutMs": { "label": "Timeout (ms)", "placeholder": "30000" },+ "autoStart": { "label": "Auto-start Server" },+ "binaryPath": { "label": "Binary Path", "placeholder": "pinchtab" },+ "startupTimeoutMs": { "label": "Startup Timeout (ms)", "placeholder": "30000" },+ "allowEvaluate": { "label": "Allow JS Evaluate" },+ "allowedDomains": { "label": "Allowed Domains" },+ "allowDownloads": { "label": "Allow Downloads" },+ "allowUploads": { "label": "Allow Uploads" },+ "defaultSnapshotFormat": { "label": "Snapshot Format" },+ "defaultSnapshotFilter": { "label": "Snapshot Filter" },+ "screenshotFormat": { "label": "Screenshot Format" },+ "screenshotQuality": { "label": "Screenshot Quality" },+ "persistSessionTabs": { "label": "Persist Tab Context" },+ "registerBrowserTool": { "label": "Register 'browser' Tool" },+ "defaultProfile": { "label": "Default Profile" },+ "profiles": { "label": "Profile Mappings" }}}diff --git a/plugin/package.json b/plugin/package.jsonindex f4fc77ad..985a8c00 100644--- a/plugin/package.json+++ b/plugin/package.json@@ -1,13 +1,21 @@{"name": "@pinchtab/pinchtab",- "version": "0.1.0",+ "version": "0.3.0","description": "OpenClaw plugin for Pinchtab browser control","type": "module",+ "scripts": {+ "test": "npx tsx --test *.test.ts"+ },"openclaw": {"extensions": ["./index.ts"]},"files": ["index.ts",+ "types.ts",+ "client.ts",+ "policy.ts",+ "session.ts",+ "tools/","openclaw.plugin.json"],"license": "MIT"diff --git a/plugin/policy.test.ts b/plugin/policy.test.tsnew file mode 100644index 00000000..316b88cb--- /dev/null+++ b/plugin/policy.test.ts@@ -0,0 +1,99 @@+import { describe, it } from "node:test";+import assert from "node:assert";+import { matchesDomain, checkNavigationPolicy, checkEvaluatePolicy, checkDownloadPolicy, checkUploadPolicy } from "./policy.ts";++describe("matchesDomain", () => {+ it("returns true for empty patterns", () => {+ assert.strictEqual(matchesDomain("https://example.com", []), true);+ });++ it("matches exact domain", () => {+ assert.strictEqual(matchesDomain("https://example.com/path", ["example.com"]), true);+ assert.strictEqual(matchesDomain("https://other.com/path", ["example.com"]), false);+ });++ it("matches wildcard subdomain", () => {+ assert.strictEqual(matchesDomain("https://sub.example.com", ["*.example.com"]), true);+ assert.strictEqual(matchesDomain("https://deep.sub.example.com", ["*.example.com"]), true);+ assert.strictEqual(matchesDomain("https://example.com", ["*.example.com"]), true);+ assert.strictEqual(matchesDomain("https://other.com", ["*.example.com"]), false);+ });++ it("handles invalid URLs", () => {+ assert.strictEqual(matchesDomain("not-a-url", ["example.com"]), false);+ });++ it("matches multiple patterns", () => {+ const patterns = ["example.com", "*.test.com"];+ assert.strictEqual(matchesDomain("https://example.com", patterns), true);+ assert.strictEqual(matchesDomain("https://sub.test.com", patterns), true);+ assert.strictEqual(matchesDomain("https://other.com", patterns), false);+ });+});++describe("checkNavigationPolicy", () => {+ it("allows navigation when no domains configured", () => {+ const result = checkNavigationPolicy({}, "https://any.com");+ assert.strictEqual(result.allowed, true);+ });++ it("allows navigation when URL matches allowedDomains", () => {+ const cfg = { allowedDomains: ["example.com"] };+ const result = checkNavigationPolicy(cfg, "https://example.com/page");+ assert.strictEqual(result.allowed, true);+ });++ it("blocks navigation when URL not in allowedDomains", () => {+ const cfg = { allowedDomains: ["example.com"] };+ const result = checkNavigationPolicy(cfg, "https://blocked.com");+ assert.strictEqual(result.allowed, false);+ assert.ok(result.error?.includes("blocked.com"));+ });++ it("allows when no URL provided", () => {+ const cfg = { allowedDomains: ["example.com"] };+ const result = checkNavigationPolicy(cfg, undefined);+ assert.strictEqual(result.allowed, true);+ });+});++describe("checkEvaluatePolicy", () => {+ it("blocks evaluate by default", () => {+ const result = checkEvaluatePolicy({});+ assert.strictEqual(result.allowed, false);+ });++ it("blocks evaluate when explicitly false", () => {+ const result = checkEvaluatePolicy({ allowEvaluate: false });+ assert.strictEqual(result.allowed, false);+ });++ it("allows evaluate when explicitly true", () => {+ const result = checkEvaluatePolicy({ allowEvaluate: true });+ assert.strictEqual(result.allowed, true);+ });+});++describe("checkDownloadPolicy", () => {+ it("blocks downloads by default", () => {+ const result = checkDownloadPolicy({});+ assert.strictEqual(result.allowed, false);+ });++ it("allows downloads when explicitly true", () => {+ const result = checkDownloadPolicy({ allowDownloads: true });+ assert.strictEqual(result.allowed, true);+ });+});++describe("checkUploadPolicy", () => {+ it("blocks uploads by default", () => {+ const result = checkUploadPolicy({});+ assert.strictEqual(result.allowed, false);+ });++ it("allows uploads when explicitly true", () => {+ const result = checkUploadPolicy({ allowUploads: true });+ assert.strictEqual(result.allowed, true);+ });+});diff --git a/plugin/policy.ts b/plugin/policy.tsnew file mode 100644index 00000000..fe14811d--- /dev/null+++ b/plugin/policy.ts@@ -0,0 +1,56 @@+import type { PluginConfig } from "./types.js";+import { textResult } from "./client.js";++export function matchesDomain(url: string, patterns: string[]): boolean {+ if (!patterns || patterns.length === 0) return true;+ try {+ const hostname = new URL(url).hostname;+ return patterns.some((p) => {+ if (p.startsWith("*.")) {+ const suffix = p.slice(2);+ return hostname === suffix || hostname.endsWith("." + suffix);+ }+ return hostname === p;+ });+ } catch {+ return false;+ }+}++export function checkNavigationPolicy(cfg: PluginConfig, url?: string): { allowed: boolean; error?: string } {+ if (!url) return { allowed: true };+ if (cfg.allowedDomains && cfg.allowedDomains.length > 0) {+ if (!matchesDomain(url, cfg.allowedDomains)) {+ return { allowed: false, error: `Navigation blocked: ${url} not in allowedDomains` };+ }+ }+ return { allowed: true };+}++export function checkEvaluatePolicy(cfg: PluginConfig): { allowed: boolean; error?: string } {+ if (cfg.allowEvaluate !== true) {+ return { allowed: false, error: "evaluate action blocked by plugin policy (allowEvaluate: false)" };+ }+ return { allowed: true };+}++export function checkDownloadPolicy(cfg: PluginConfig): { allowed: boolean; error?: string } {+ if (cfg.allowDownloads !== true) {+ return { allowed: false, error: "downloads blocked by plugin policy (allowDownloads: false)" };+ }+ return { allowed: true };+}++export function checkUploadPolicy(cfg: PluginConfig): { allowed: boolean; error?: string } {+ if (cfg.allowUploads !== true) {+ return { allowed: false, error: "uploads blocked by plugin policy (allowUploads: false)" };+ }+ return { allowed: true };+}++export function enforcePolicyOrReturn(check: { allowed: boolean; error?: string }): any | null {+ if (!check.allowed) {+ return textResult({ error: check.error });+ }+ return null;+}diff --git a/plugin/session.test.ts b/plugin/session.test.tsnew file mode 100644index 00000000..10ba1bb6--- /dev/null+++ b/plugin/session.test.ts@@ -0,0 +1,65 @@+import { describe, it, beforeEach } from "node:test";+import assert from "node:assert";+import { getLastTabId, setLastTabId, resolveProfile } from "./session.ts";++describe("tab session state", () => {+ beforeEach(() => {+ setLastTabId(undefined);+ });++ it("starts with undefined tabId", () => {+ assert.strictEqual(getLastTabId(), undefined);+ });++ it("stores and retrieves tabId", () => {+ setLastTabId("tab123");+ assert.strictEqual(getLastTabId(), "tab123");+ });++ it("can clear tabId", () => {+ setLastTabId("tab123");+ setLastTabId(undefined);+ assert.strictEqual(getLastTabId(), undefined);+ });+});++describe("resolveProfile", () => {+ it("returns empty object for default openclaw profile", () => {+ const result = resolveProfile({}, undefined);+ assert.deepStrictEqual(result, {});+ });++ it("returns attach:true for user profile", () => {+ const result = resolveProfile({}, "user");+ assert.deepStrictEqual(result, { attach: true });+ });++ it("uses config defaultProfile", () => {+ const cfg = { defaultProfile: "user" };+ const result = resolveProfile(cfg, undefined);+ assert.deepStrictEqual(result, { attach: true });+ });++ it("returns custom profile config", () => {+ const cfg = {+ profiles: {+ staging: { instanceId: "staging-instance" },+ custom: { instanceId: "custom-id", attach: true },+ },+ };+ assert.deepStrictEqual(resolveProfile(cfg, "staging"), { instanceId: "staging-instance" });+ assert.deepStrictEqual(resolveProfile(cfg, "custom"), { instanceId: "custom-id", attach: true });+ });++ it("falls back to builtin for unknown profile", () => {+ const cfg = { profiles: { staging: { instanceId: "staging" } } };+ const result = resolveProfile(cfg, "unknown");+ assert.deepStrictEqual(result, {});+ });++ it("profile param overrides defaultProfile", () => {+ const cfg = { defaultProfile: "user", profiles: { staging: { instanceId: "s1" } } };+ const result = resolveProfile(cfg, "staging");+ assert.deepStrictEqual(result, { instanceId: "s1" });+ });+});diff --git a/plugin/session.ts b/plugin/session.tsnew file mode 100644index 00000000..08b007a1--- /dev/null+++ b/plugin/session.ts@@ -0,0 +1,96 @@+import type { PluginConfig } from "./types.js";+import { pinchtabFetch } from "./client.js";++let serverStarted = false;+let lastTabId: string | undefined;++export function getLastTabId(): string | undefined {+ return lastTabId;+}++export function setLastTabId(tabId: string | undefined): void {+ lastTabId = tabId;+}++export function resolveProfile(cfg: PluginConfig, profile?: string): { instanceId?: string; attach?: boolean } {+ const name = profile || cfg.defaultProfile || "openclaw";+ if (cfg.profiles?.[name]) {+ return cfg.profiles[name];+ }+ if (name === "user") {+ return { attach: true };+ }+ return {};+}++export async function getEnhancedHealth(cfg: PluginConfig): Promise<any> {+ const base = cfg.baseUrl || "http://localhost:9867";+ const health = await pinchtabFetch(cfg, "/health");+ const serverOk = !health?.error;++ const result: any = {+ server: serverOk ? "ok" : "unreachable",+ baseUrl: base,+ autoStart: cfg.autoStart !== false,+ defaultProfile: cfg.defaultProfile || "openclaw",+ policies: {+ allowEvaluate: cfg.allowEvaluate === true,+ allowDownloads: cfg.allowDownloads === true,+ allowUploads: cfg.allowUploads === true,+ allowedDomains: cfg.allowedDomains?.length ? cfg.allowedDomains : "all",+ },+ };++ if (serverOk) {+ result.serverHealth = health;+ if (health?.version) result.serverVersion = health.version;+ } else {+ result.error = health?.error;+ }++ const warnings: string[] = [];+ if (cfg.allowEvaluate) warnings.push("evaluate enabled - JS execution allowed");+ if (!cfg.allowedDomains?.length) warnings.push("no domain restrictions");+ if (warnings.length) result.warnings = warnings;++ return result;+}++export async function ensureServerRunning(cfg: PluginConfig): Promise<{ ok: boolean; error?: string }> {+ const base = cfg.baseUrl || "http://localhost:9867";+ const isLocal = base.includes("localhost") || base.includes("127.0.0.1");++ const healthCheck = await pinchtabFetch(cfg, "/health");+ if (!healthCheck?.error) {+ return { ok: true };+ }++ if (!isLocal || cfg.autoStart === false || serverStarted) {+ return { ok: false, error: healthCheck.error };+ }++ const binary = cfg.binaryPath || "pinchtab";+ const startupTimeout = cfg.startupTimeoutMs ?? 30000;++ try {+ const { spawn } = await import("child_process");+ const proc = spawn(binary, ["server"], {+ detached: true,+ stdio: "ignore",+ });+ proc.unref();+ serverStarted = true;++ const startTime = Date.now();+ while (Date.now() - startTime < startupTimeout) {+ await new Promise((r) => setTimeout(r, 500));+ const check = await pinchtabFetch(cfg, "/health");+ if (!check?.error) {+ return { ok: true };+ }+ }+ return { ok: false, error: `Server failed to start within ${startupTimeout}ms` };+ } catch (err: any) {+ return { ok: false, error: `Failed to start pinchtab: ${err?.message}` };+ }+}diff --git a/plugin/tools/browser.ts b/plugin/tools/browser.tsnew file mode 100644index 00000000..e07747ed--- /dev/null+++ b/plugin/tools/browser.ts@@ -0,0 +1,163 @@+import type { PluginConfig } from "../types.js";+import { pinchtabFetch, textResult, imageResult, resourceResult } from "../client.js";+import { checkNavigationPolicy, enforcePolicyOrReturn } from "../policy.js";+import { ensureServerRunning, getEnhancedHealth, resolveProfile, getLastTabId, setLastTabId } from "../session.js";++export const browserToolSchema = {+ type: "object",+ properties: {+ action: {+ type: "string",+ enum: ["navigate", "snapshot", "screenshot", "click", "type", "fill", "press", "hover", "scroll", "select", "tabs", "pdf", "status"],+ description: "Browser action",+ },+ url: { type: "string", description: "URL to navigate to" },+ profile: { type: "string", description: "Profile: 'openclaw' (isolated) or 'user' (attached session)" },+ ref: { type: "string", description: "Element ref from snapshot (e.g. e5)" },+ text: { type: "string", description: "Text to type or fill" },+ key: { type: "string", description: "Key to press (Enter, Tab, Escape)" },+ value: { type: "string", description: "Value for select" },+ selector: { type: "string", description: "CSS selector for snapshot scope" },+ format: { type: "string", enum: ["compact", "json", "text"], description: "Output format" },+ maxTokens: { type: "number", description: "Truncate output to ~N tokens" },+ quality: { type: "number", description: "Screenshot JPEG quality 1-100" },+ tabId: { type: "string", description: "Target tab ID" },+ tabAction: { type: "string", enum: ["list", "new", "close"], description: "Tab operation (for action=tabs)" },+ newTab: { type: "boolean", description: "Open URL in new tab" },+ landscape: { type: "boolean", description: "PDF landscape orientation" },+ scale: { type: "number", description: "PDF print scale" },+ },+ required: ["action"],+};++export const browserToolDescription = `OpenClaw-compatible browser control (backed by Pinchtab).++Actions:+- navigate: go to URL (url, profile?, newTab?)+- snapshot: accessibility tree for interactions (selector?, format?, maxTokens?)+- screenshot: capture page image (quality?, format?)+- click/type/fill/press/hover/scroll/select: element actions (ref, text?, key?, value?)+- tabs: list/new/close tabs (tabAction?, url?, tabId?)+- pdf: export page as PDF (landscape?, scale?)+- status: check browser/server health++Profiles: "openclaw" (default isolated), "user" (attach to existing session)`;++export async function executeBrowserAction(cfg: PluginConfig, params: any): Promise<any> {+ const { action, profile } = params;++ // Resolve profile to instance+ const profileConfig = resolveProfile(cfg, profile);++ // Auto-start server if needed+ if (action !== "status") {+ const serverCheck = await ensureServerRunning(cfg);+ if (!serverCheck.ok) {+ return textResult({ error: serverCheck.error });+ }+ }++ // Session tab persistence+ if (cfg.persistSessionTabs !== false && !params.tabId && getLastTabId()) {+ params.tabId = getLastTabId();+ }++ // --- navigate ---+ if (action === "navigate") {+ const navPolicy = enforcePolicyOrReturn(checkNavigationPolicy(cfg, params.url));+ if (navPolicy) return navPolicy;++ const body: any = { url: params.url };+ if (params.tabId) body.tabId = params.tabId;+ if (params.newTab) body.newTab = true;+ if (profileConfig.instanceId) body.instanceId = profileConfig.instanceId;+ const result = await pinchtabFetch(cfg, "/navigate", { body });+ if (result?.tabId) setLastTabId(result.tabId);+ return textResult(result);+ }++ // --- snapshot ---+ if (action === "snapshot") {+ const query = new URLSearchParams();+ if (params.tabId) query.set("tabId", params.tabId);+ query.set("filter", "interactive");+ query.set("format", params.format || cfg.defaultSnapshotFormat || "compact");+ if (params.selector) query.set("selector", params.selector);+ if (params.maxTokens) query.set("maxTokens", String(params.maxTokens));+ return textResult(await pinchtabFetch(cfg, `/snapshot?${query.toString()}`));+ }++ // --- screenshot ---+ if (action === "screenshot") {+ const query = new URLSearchParams();+ if (params.tabId) query.set("tabId", params.tabId);+ const fmt = params.format === "png" ? "png" : (cfg.screenshotFormat || "jpeg");+ if (fmt === "png") query.set("format", "png");+ query.set("quality", String(params.quality || cfg.screenshotQuality || 80));+ try {+ const res = await pinchtabFetch(cfg, `/screenshot?${query.toString()}`, { rawResponse: true });+ if (res instanceof Response) {+ if (!res.ok) return textResult({ error: `Screenshot failed: ${res.status}` });+ const buf = await res.arrayBuffer();+ const b64 = Buffer.from(buf).toString("base64");+ return imageResult(b64, fmt === "png" ? "image/png" : "image/jpeg");+ }+ return textResult(res);+ } catch (err: any) {+ return textResult({ error: `Screenshot failed: ${err?.message}` });+ }+ }++ // --- element actions ---+ if (["click", "type", "fill", "press", "hover", "scroll", "select"].includes(action)) {+ const body: any = { kind: action };+ if (params.ref) body.ref = params.ref;+ if (params.text) body.text = params.text;+ if (params.key) body.key = params.key;+ if (params.value) body.value = params.value;+ if (params.tabId) body.tabId = params.tabId;+ return textResult(await pinchtabFetch(cfg, "/action", { body }));+ }++ // --- tabs ---+ if (action === "tabs") {+ const tabAction = params.tabAction || "list";+ if (tabAction === "list") {+ return textResult(await pinchtabFetch(cfg, "/tabs"));+ }+ const body: any = { action: tabAction };+ if (params.url) body.url = params.url;+ if (params.tabId) body.tabId = params.tabId;+ const result = await pinchtabFetch(cfg, "/tab", { body });+ if (tabAction === "new" && result?.tabId) setLastTabId(result.tabId);+ return textResult(result);+ }++ // --- pdf ---+ if (action === "pdf") {+ const query = new URLSearchParams();+ query.set("raw", "true");+ if (params.tabId) query.set("tabId", params.tabId);+ if (params.landscape) query.set("landscape", "true");+ if (params.scale) query.set("scale", String(params.scale));+ try {+ const res = await pinchtabFetch(cfg, `/pdf?${query.toString()}`, { rawResponse: true });+ if (res instanceof Response) {+ if (!res.ok) return textResult({ error: `PDF failed: ${res.status}` });+ const buf = await res.arrayBuffer();+ const b64 = Buffer.from(buf).toString("base64");+ return resourceResult("pdf://export", "application/pdf", b64);+ }+ return textResult(res);+ } catch (err: any) {+ return textResult({ error: `PDF failed: ${err?.message}` });+ }+ }++ // --- status ---+ if (action === "status") {+ return textResult(await getEnhancedHealth(cfg));+ }++ return textResult({ error: `Unknown browser action: ${action}` });+}diff --git a/plugin/tools/pinchtab.ts b/plugin/tools/pinchtab.tsnew file mode 100644index 00000000..8a482e43--- /dev/null+++ b/plugin/tools/pinchtab.ts@@ -0,0 +1,365 @@+import type { PluginConfig } from "../types.js";+import { pinchtabFetch, textResult, imageResult, resourceResult, normalizeActionParams, looksLikeStaleRef } from "../client.js";+import { checkNavigationPolicy, checkEvaluatePolicy, checkDownloadPolicy, checkUploadPolicy, enforcePolicyOrReturn } from "../policy.js";+import { ensureServerRunning, getEnhancedHealth, getLastTabId, setLastTabId } from "../session.js";++export const pinchtabToolSchema = {+ type: "object",+ properties: {+ action: {+ type: "string",+ enum: [+ "navigate", "snapshot", "click", "type", "press", "fill", "hover",+ "mouse-move", "mouse-down", "mouse-up", "mouse-wheel", "scroll",+ "select", "focus", "text", "wait", "handoff", "tabs", "screenshot",+ "evaluate", "pdf", "download", "upload", "network", "health",+ ],+ description: "Action to perform",+ },+ url: { type: "string", description: "URL for navigate or new tab" },+ ref: { type: "string", description: "Element ref from snapshot (e.g. e5)" },+ text: { type: "string", description: "Text to type or fill" },+ key: { type: "string", description: "Key to press (e.g. Enter, Tab, Escape)" },+ expression: { type: "string", description: "JavaScript expression for evaluate" },+ selector: { type: "string", description: "CSS selector for snapshot scope or action target" },+ filter: { type: "string", enum: ["interactive", "all"], description: "Snapshot filter" },+ format: { type: "string", enum: ["json", "compact", "text", "yaml"], description: "Snapshot format" },+ maxTokens: { type: "number", description: "Truncate snapshot to ~N tokens" },+ depth: { type: "number", description: "Max snapshot tree depth" },+ diff: { type: "boolean", description: "Snapshot diff: only changes since last" },+ value: { type: "string", description: "Value for select dropdown" },+ scrollY: { type: "number", description: "Pixels to scroll vertically" },+ x: { type: "number", description: "Mouse X coordinate" },+ y: { type: "number", description: "Mouse Y coordinate" },+ button: { type: "string", enum: ["left", "right", "middle"], description: "Mouse button" },+ deltaX: { type: "number", description: "Mouse wheel horizontal delta" },+ deltaY: { type: "number", description: "Mouse wheel vertical delta" },+ waitNav: { type: "boolean", description: "Wait for navigation after action" },+ tabId: { type: "string", description: "Target tab ID" },+ tabAction: { type: "string", enum: ["list", "new", "close"], description: "Tab sub-action" },+ newTab: { type: "boolean", description: "Open URL in new tab" },+ blockImages: { type: "boolean", description: "Block image loading" },+ timeout: { type: "number", description: "Navigation timeout in seconds" },+ quality: { type: "number", description: "JPEG quality 1-100" },+ screenshotFormat: { type: "string", enum: ["jpeg", "png"], description: "Screenshot format" },+ mode: { type: "string", enum: ["readability", "raw"], description: "Text extraction mode" },+ ms: { type: "number", description: "Wait milliseconds" },+ state: { type: "string", enum: ["visible", "hidden", "attached", "detached"], description: "Wait state" },+ load: { type: "string", enum: ["load", "domcontentloaded", "networkidle"], description: "Document load state" },+ fn: { type: "string", description: "JavaScript predicate for wait" },+ humanReason: { type: "string", description: "Reason for manual handoff" },+ humanPrompt: { type: "string", description: "Instruction for human handoff" },+ landscape: { type: "boolean", description: "PDF landscape orientation" },+ scale: { type: "number", description: "PDF print scale" },+ downloadUrl: { type: "string", description: "URL to download file from" },+ savePath: { type: "string", description: "Path to save downloaded file" },+ files: { type: "array", items: { type: "string" }, description: "Base64-encoded files for upload" },+ paths: { type: "array", items: { type: "string" }, description: "File paths for upload" },+ networkAction: { type: "string", enum: ["list", "get", "export", "clear"], description: "Network sub-action" },+ requestId: { type: "string", description: "Network request ID for get action" },+ method: { type: "string", description: "Filter network by HTTP method" },+ status: { type: "string", description: "Filter network by status (e.g. 4xx, 5xx, 200)" },+ resourceType: { type: "string", description: "Filter network by resource type (xhr, fetch, document)" },+ limit: { type: "number", description: "Limit network results" },+ exportFormat: { type: "string", enum: ["har", "json"], description: "Network export format" },+ },+ required: ["action"],+};++export const pinchtabToolDescription = `Browser control via Pinchtab. Actions:+- navigate: go to URL (url, tabId?, newTab?, blockImages?, timeout?)+- snapshot: accessibility tree (filter?, format?, selector?, maxTokens?, depth?, diff?, tabId?)+- click/type/press/fill/hover/scroll/select/focus: act on element (ref, text?, key?, value?, scrollY?, waitNav?, tabId?)+- mouse-move/mouse-down/mouse-up/mouse-wheel: low-level mouse controls (ref|selector|x+y, button?, deltaX?, deltaY?, tabId?)+- text: extract readable text (mode?, tabId?)+- wait: pause until condition (selector|text|url|load|fn|ms, tabId?, timeout?, state?)+- handoff: request human intervention mid-flow (captcha/login/2FA/credentials)+- tabs: list/new/close tabs (tabAction?, url?, tabId?)+- screenshot: JPEG screenshot (quality?, tabId?)+- evaluate: run JS (expression, tabId?)+- pdf: export page as PDF (landscape?, scale?, tabId?)+- download: download file from URL (downloadUrl, savePath?, tabId?)+- upload: upload files to file input (selector, files[]|paths[], tabId?)+- network: capture/inspect network requests (networkAction: list|get|export|clear, requestId?, method?, status?, limit?)+- health: check connectivity++Token strategy: use "text" for reading (~800 tokens), "snapshot" with filter=interactive&format=compact for interactions (~3,600), diff=true on subsequent snapshots.`;++export async function executePinchtabAction(cfg: PluginConfig, params: any): Promise<any> {+ const normalized = normalizeActionParams(params);+ const { action } = normalized;++ // Auto-start server if needed+ if (action !== "health") {+ const serverCheck = await ensureServerRunning(cfg);+ if (!serverCheck.ok) {+ return textResult({ error: serverCheck.error });+ }+ }++ // Session tab persistence+ if (cfg.persistSessionTabs !== false && !normalized.tabId && getLastTabId()) {+ normalized.tabId = getLastTabId();+ }++ // Policy: block evaluate if not allowed+ if (action === "evaluate") {+ const policyResult = enforcePolicyOrReturn(checkEvaluatePolicy(cfg));+ if (policyResult) return policyResult;+ }++ // --- navigate ---+ if (action === "navigate") {+ const navPolicy = enforcePolicyOrReturn(checkNavigationPolicy(cfg, normalized.url));+ if (navPolicy) return navPolicy;++ const body: any = { url: normalized.url };+ if (normalized.tabId) body.tabId = normalized.tabId;+ if (normalized.newTab) body.newTab = true;+ if (normalized.blockImages) body.blockImages = true;+ if (normalized.timeout) body.timeout = normalized.timeout;+ const result = await pinchtabFetch(cfg, "/navigate", { body });+ if (result?.tabId) setLastTabId(result.tabId);+ return textResult(result);+ }++ // --- snapshot ---+ if (action === "snapshot") {+ const query = new URLSearchParams();+ if (normalized.tabId) query.set("tabId", normalized.tabId);+ const filter = normalized.filter ?? cfg.defaultSnapshotFilter ?? "interactive";+ const format = normalized.format ?? cfg.defaultSnapshotFormat ?? "compact";+ query.set("filter", filter);+ query.set("format", format);+ if (normalized.selector) query.set("selector", normalized.selector);+ if (normalized.maxTokens) query.set("maxTokens", String(normalized.maxTokens));+ if (normalized.depth) query.set("depth", String(normalized.depth));+ if (normalized.diff) query.set("diff", "true");+ return textResult(await pinchtabFetch(cfg, `/snapshot?${query.toString()}`));+ }++ // --- element actions ---+ const elementActions = [+ "click", "type", "press", "fill", "hover", "mouse-move", "mouse-down",+ "mouse-up", "mouse-wheel", "scroll", "select", "focus",+ ];+ if (elementActions.includes(action)) {+ const body: any = { kind: action };+ for (const k of ["ref", "text", "key", "selector", "value", "scrollY", "x", "y", "button", "deltaX", "deltaY", "tabId", "waitNav"]) {+ if (normalized[k] !== undefined) body[k] = normalized[k];+ }++ let result = await pinchtabFetch(cfg, "/action", { body });++ // Stale ref recovery+ if (body.ref && looksLikeStaleRef(result)) {+ const q = new URLSearchParams();+ q.set("filter", "interactive");+ q.set("format", "compact");+ if (body.tabId) q.set("tabId", body.tabId);+ await pinchtabFetch(cfg, `/snapshot?${q.toString()}`);+ const retried = await pinchtabFetch(cfg, "/action", { body });+ result = retried?.error+ ? { ...retried, warning: "Action retried once after snapshot refresh but still failed." }+ : { ...retried, warning: "Action succeeded after automatic snapshot refresh (stale ref recovery)." };+ }++ // Fill fallback to type+ if (action === "fill" && result?.error && body.ref && (typeof body.text === "string" || typeof body.value === "string")) {+ const typeBody: any = { ...body, kind: "type" };+ if (typeof typeBody.text !== "string" && typeof typeBody.value === "string") {+ typeBody.text = typeBody.value;+ }+ delete typeBody.value;+ const typed = await pinchtabFetch(cfg, "/action", { body: typeBody });+ if (!typed?.error) {+ result = { ...typed, warning: "Fill failed; type fallback succeeded." };+ }+ }++ return textResult(result);+ }++ // --- text ---+ if (action === "text") {+ const query = new URLSearchParams();+ if (normalized.tabId) query.set("tabId", normalized.tabId);+ if (normalized.mode) query.set("mode", normalized.mode);+ return textResult(await pinchtabFetch(cfg, `/text?${query.toString()}`));+ }++ // --- wait ---+ if (action === "wait") {+ const body: any = {};+ for (const k of ["selector", "text", "url", "load", "fn", "ms", "tabId", "timeout", "state"]) {+ if (normalized[k] !== undefined) body[k] = normalized[k];+ }+ return textResult(await pinchtabFetch(cfg, "/wait", { body }));+ }++ // --- handoff ---+ if (action === "handoff") {+ const hasWaitCondition = ["selector", "text", "url", "load", "fn", "ms"].some((k) => normalized[k] !== undefined);+ const handoffMeta = {+ status: "human_handoff_required",+ reason: normalized.humanReason || "Manual intervention required (captcha/login/2FA/credential entry).",+ instructions: normalized.humanPrompt || "Please complete the step in the headed browser, then resume automation.",+ };++ if (!hasWaitCondition) {+ return textResult({ ...handoffMeta, resumed: false, next: "Call with a wait condition or use action='wait' to resume." });+ }++ const waitBody: any = {};+ for (const k of ["selector", "text", "url", "load", "fn", "ms", "tabId", "timeout", "state"]) {+ if (normalized[k] !== undefined) waitBody[k] = normalized[k];+ }+ const waitResult = await pinchtabFetch(cfg, "/wait", { body: waitBody });+ return textResult({ ...handoffMeta, resumed: !waitResult?.error, waitResult });+ }++ // --- tabs ---+ if (action === "tabs") {+ const tabAction = normalized.tabAction || "list";+ if (tabAction === "list") {+ const listed = await pinchtabFetch(cfg, "/tabs");+ const tabs = Array.isArray(listed?.tabs) ? listed.tabs : Array.isArray(listed) ? listed : [];+ if (tabs.length > 0) return textResult(listed);++ const instances = await pinchtabFetch(cfg, "/instances");+ const list = Array.isArray(instances?.value) ? instances.value : Array.isArray(instances) ? instances : [];+ const running = list.find((i: any) => i?.status === "running" && i?.id);+ if (!running) {+ return textResult({ ...listed, warning: "No tabs returned and no running instance found." });+ }+ const instanceTabs = await pinchtabFetch(cfg, `/instances/${running.id}/tabs`);+ return textResult({ source: "instance-fallback", instanceId: running.id, tabs: instanceTabs?.tabs ?? instanceTabs });+ }+ const body: any = { action: tabAction };+ if (normalized.url) body.url = normalized.url;+ if (normalized.tabId) body.tabId = normalized.tabId;+ const result = await pinchtabFetch(cfg, "/tab", { body });+ if (tabAction === "new" && result?.tabId) setLastTabId(result.tabId);+ return textResult(result);+ }++ // --- screenshot ---+ if (action === "screenshot") {+ const query = new URLSearchParams();+ if (normalized.tabId) query.set("tabId", normalized.tabId);+ const format = normalized.screenshotFormat ?? cfg.screenshotFormat ?? "jpeg";+ const quality = normalized.quality ?? cfg.screenshotQuality ?? 80;+ if (format === "png") query.set("format", "png");+ query.set("quality", String(quality));+ try {+ const res = await pinchtabFetch(cfg, `/screenshot?${query.toString()}`, { rawResponse: true });+ if (res instanceof Response) {+ if (!res.ok) return textResult({ error: `Screenshot failed: ${res.status} ${await res.text()}` });+ const buf = await res.arrayBuffer();+ const b64 = Buffer.from(buf).toString("base64");+ return imageResult(b64, format === "png" ? "image/png" : "image/jpeg");+ }+ return textResult(res);+ } catch (err: any) {+ return textResult({ error: `Screenshot failed: ${err?.message}` });+ }+ }++ // --- evaluate ---+ if (action === "evaluate") {+ const body: any = { expression: normalized.expression };+ if (normalized.tabId) body.tabId = normalized.tabId;+ return textResult(await pinchtabFetch(cfg, "/evaluate", { body }));+ }++ // --- pdf ---+ if (action === "pdf") {+ const query = new URLSearchParams();+ query.set("raw", "true");+ if (normalized.tabId) query.set("tabId", normalized.tabId);+ if (normalized.landscape) query.set("landscape", "true");+ if (normalized.scale) query.set("scale", String(normalized.scale));+ try {+ const res = await pinchtabFetch(cfg, `/pdf?${query.toString()}`, { rawResponse: true });+ if (res instanceof Response) {+ if (!res.ok) return textResult({ error: `PDF failed: ${res.status} ${await res.text()}` });+ const buf = await res.arrayBuffer();+ const b64 = Buffer.from(buf).toString("base64");+ return resourceResult("pdf://export", "application/pdf", b64);+ }+ return textResult(res);+ } catch (err: any) {+ return textResult({ error: `PDF failed: ${err?.message}` });+ }+ }++ // --- network ---+ if (action === "network") {+ const networkAction = normalized.networkAction || "list";++ if (networkAction === "clear") {+ const query = new URLSearchParams();+ if (normalized.tabId) query.set("tabId", normalized.tabId);+ return textResult(await pinchtabFetch(cfg, `/network/clear?${query.toString()}`, { method: "POST" }));+ }++ if (networkAction === "get" && normalized.requestId) {+ return textResult(await pinchtabFetch(cfg, `/network/${normalized.requestId}`));+ }++ if (networkAction === "export") {+ const query = new URLSearchParams();+ if (normalized.tabId) query.set("tabId", normalized.tabId);+ if (normalized.exportFormat === "json") query.set("format", "json");+ const result = await pinchtabFetch(cfg, `/network/export?${query.toString()}`);+ if (normalized.exportFormat === "har" || !normalized.exportFormat) {+ return resourceResult("har://export", "application/json", Buffer.from(JSON.stringify(result)).toString("base64"));+ }+ return textResult(result);+ }++ // list+ const query = new URLSearchParams();+ if (normalized.tabId) query.set("tabId", normalized.tabId);+ if (normalized.method) query.set("method", normalized.method);+ if (normalized.status) query.set("status", normalized.status);+ if (normalized.resourceType) query.set("type", normalized.resourceType);+ if (normalized.limit) query.set("limit", String(normalized.limit));+ return textResult(await pinchtabFetch(cfg, `/network?${query.toString()}`));+ }++ // --- download ---+ if (action === "download") {+ const policyResult = enforcePolicyOrReturn(checkDownloadPolicy(cfg));+ if (policyResult) return policyResult;++ const query = new URLSearchParams();+ if (normalized.downloadUrl) query.set("url", normalized.downloadUrl);+ if (normalized.tabId) query.set("tabId", normalized.tabId);+ if (normalized.savePath) {+ query.set("output", "file");+ query.set("path", normalized.savePath);+ }+ return textResult(await pinchtabFetch(cfg, `/download?${query.toString()}`));+ }++ // --- upload ---+ if (action === "upload") {+ const policyResult = enforcePolicyOrReturn(checkUploadPolicy(cfg));+ if (policyResult) return policyResult;++ const body: any = {};+ if (normalized.tabId) body.tabId = normalized.tabId;+ if (normalized.selector) body.selector = normalized.selector;+ if (normalized.files) body.files = normalized.files;+ if (normalized.paths) body.paths = normalized.paths;+ return textResult(await pinchtabFetch(cfg, "/upload", { body }));+ }++ // --- health ---+ if (action === "health") {+ return textResult(await getEnhancedHealth(cfg));+ }++ return textResult({ error: `Unknown action: ${action}` });+}diff --git a/plugin/types.ts b/plugin/types.tsnew file mode 100644index 00000000..23f8c51f--- /dev/null+++ b/plugin/types.ts@@ -0,0 +1,35 @@+export interface PluginConfig {+ baseUrl?: string;+ token?: string;+ timeoutMs?: number;+ /** @deprecated Use timeoutMs instead */+ timeout?: number;+ autoStart?: boolean;+ binaryPath?: string;+ startupTimeoutMs?: number;+ allowEvaluate?: boolean;+ allowedDomains?: string[];+ allowDownloads?: boolean;+ allowUploads?: boolean;+ defaultSnapshotFormat?: string;+ defaultSnapshotFilter?: string;+ screenshotFormat?: string;+ screenshotQuality?: number;+ persistSessionTabs?: boolean;+ registerBrowserTool?: boolean;+ defaultProfile?: string;+ profiles?: Record<string, { instanceId?: string; attach?: boolean }>;+}++export interface PluginApi {+ config: { plugins?: { entries?: Record<string, { config?: PluginConfig }> } };+ registerTool: (tool: any, opts?: { optional?: boolean }) => void;+}++export interface ToolResult {+ content: Array<+ | { type: "text"; text: string }+ | { type: "image"; data: string; mimeType: string }+ | { type: "resource"; resource: { uri: string; mimeType: string; blob: string } }+ >;+}diff --git a/scripts/check-plugin.sh b/scripts/check-plugin.shnew file mode 100755index 00000000..7b9d9bd5--- /dev/null+++ b/scripts/check-plugin.sh@@ -0,0 +1,23 @@+#!/usr/bin/env bash+set -euo pipefail++cd "$(dirname "$0")/.."++echo "📦 Plugin checks"+echo ""++# Verify JSON files+echo " Validating JSON schemas..."+node -e "JSON.parse(require('fs').readFileSync('plugin/package.json', 'utf8'))"+node -e "JSON.parse(require('fs').readFileSync('plugin/openclaw.plugin.json', 'utf8'))"+echo " ✓ JSON valid"++# Verify package contents+echo ""+echo " Verifying package contents..."+cd plugin+npm pack --dry-run 2>&1 | grep -E "^npm notice [0-9]" | awk '{print " " $4 " (" $3 ")"}'+echo " ✓ Package verified"++echo ""+echo "✅ Plugin checks passed"diff --git a/scripts/e2e.sh b/scripts/e2e.shindex 32e77776..315aa9ad 100755--- a/scripts/e2e.sh+++ b/scripts/e2e.sh@@ -627,10 +627,46 @@ run_pr() {return 0}+run_plugin() {+ local compose_file="tests/e2e/docker-compose.yml"+ local summary_file="tests/e2e/results/summary-plugin.txt"+ local report_file="tests/e2e/results/report-plugin.md"+ local progress_file="tests/e2e/results/progress-plugin.log"+ local log_prefix="logs-plugin"+ local output_file="tests/e2e/results/output-plugin.log"+ echo " ${ACCENT}${BOLD}E2E Plugin tests (Docker)${NC}"+ show_filter_status+ show_logs_status+ echo ""+ prepare_suite_results "${summary_file}" "${report_file}" "${progress_file}" "${log_prefix}" "${output_file}"+ local suite_started_at+ suite_started_at=$(now_ms)+ set +e+ run_logged_command "${output_file}" "" "building support images" build_support_images "${compose_file}"+ local plugin_exit=$?+ local -a args=()+ [ -n "${E2E_FILTER}" ] && args+=("filter=${E2E_FILTER}")+ if [ "${plugin_exit}" -eq 0 ]; then+ run_logged_command "${output_file}" "${progress_file}" "running plugin suite" compose -f "${compose_file}" run --build --rm runner-api /bin/bash /e2e/run.sh plugin "${args[@]}"+ plugin_exit=$?+ fi+ set -e+ local suite_duration_ms=$(( $(now_ms) - suite_started_at ))+ record_suite_duration "${summary_file}" "${report_file}" "${suite_duration_ms}"+ if [ "${plugin_exit}" -ne 0 ]; then+ dump_compose_failure "${compose_file}" "${log_prefix}" runner-api pinchtab+ show_failure_summary "${report_file}" "${output_file}"+ show_suite_artifacts "${summary_file}" "${report_file}" "${progress_file}" "${log_prefix}" "${output_file}" "${suite_duration_ms}" runner-api pinchtab+ fi+ compose_down "${compose_file}"+ return "${plugin_exit}"+}+run_release() {local api_exit=0local cli_exit=0local infra_exit=0+ local plugin_exit=0local ran_any=0if suite_filter_matches "tests/e2e/scenarios/api" true; then@@ -658,14 +694,20 @@ run_release() {show_suite_skip "infra-extended"fi+ echo ""++ # Plugin smoke tests (always run in release suite)+ ran_any=1+ run_plugin || plugin_exit=$?+echo ""if [ "${ran_any}" -eq 0 ]; thenecho " ${ERROR}No E2E suites matched filter '${E2E_FILTER}'${NC}"return 1fi- if [ "${api_exit}" -ne 0 ] || [ "${cli_exit}" -ne 0 ] || [ "${infra_exit}" -ne 0 ]; then+ if [ "${api_exit}" -ne 0 ] || [ "${cli_exit}" -ne 0 ] || [ "${infra_exit}" -ne 0 ] || [ "${plugin_exit}" -ne 0 ]; thenecho " ${ERROR}Some E2E suites failed${NC}"- echo " ${MUTED}exit codes: api-extended=${api_exit}, cli-extended=${cli_exit}, infra-extended=${infra_exit}${NC}"+ echo " ${MUTED}exit codes: api-extended=${api_exit}, cli-extended=${cli_exit}, infra-extended=${infra_exit}, plugin=${plugin_exit}${NC}"return 1fiecho " ${SUCCESS}All E2E suites passed${NC}"@@ -698,6 +740,9 @@ case "${suite}" ininfra-extended)run_infra_extended;;+ plugin)+ run_plugin+ ;;release|all)run_release;;@@ -716,7 +761,7 @@ case "${suite}" in;;*)echo "Unknown E2E suite: ${suite}" >&2- echo "Available suites: pr, api, cli, infra, api-extended, cli-extended, infra-extended, release" >&2+ echo "Available suites: pr, api, cli, infra, plugin, api-extended, cli-extended, infra-extended, release" >&2exit 1;;esacdiff --git a/tests/e2e/run.sh b/tests/e2e/run.shindex c83bc288..b54c82b0 100644--- a/tests/e2e/run.sh+++ b/tests/e2e/run.sh@@ -95,9 +95,24 @@ case "$SUITE" inPROGRESS_FILE_EXTENDED="progress-infra-extended.log"REQUIRED_COMMANDS=(curl jq grep sed awk seq);;+ plugin)+ source "${ROOT_DIR}/helpers/api.sh"+ GROUP_DIR="${ROOT_DIR}/scenarios/plugin"+ SUITE_KIND="api"+ RUN_EXTENDED=false+ SUITE_TITLE_BASIC="PinchTab E2E Plugin Suite"+ SUITE_TITLE_EXTENDED="PinchTab E2E Plugin Suite"+ SUMMARY_FILE_BASIC="summary-plugin.txt"+ SUMMARY_FILE_EXTENDED="summary-plugin.txt"+ REPORT_FILE_BASIC="report-plugin.md"+ REPORT_FILE_EXTENDED="report-plugin.md"+ PROGRESS_FILE_BASIC="progress-plugin.log"+ PROGRESS_FILE_EXTENDED="progress-plugin.log"+ REQUIRED_COMMANDS=(curl jq grep sed awk seq)+ ;;*)echo "unknown suite: $SUITE" >&2- echo "usage: /bin/bash tests/e2e/run.sh api|cli|infra [extended=true|all=true] [filter=<substring>] [extra=<files>]" >&2+ echo "usage: /bin/bash tests/e2e/run.sh api|cli|infra|plugin [extended=true|all=true] [filter=<substring>] [extra=<files>]" >&2exit 1;;esacdiff --git a/tests/e2e/scenarios/plugin/plugin-basic.sh b/tests/e2e/scenarios/plugin/plugin-basic.shnew file mode 100644index 00000000..9fee653e--- /dev/null+++ b/tests/e2e/scenarios/plugin/plugin-basic.sh@@ -0,0 +1,203 @@+#!/bin/bash+# plugin-basic.sh — Tests core endpoints used by the OpenClaw plugin.++GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+source "${GROUP_DIR}/../../helpers/api.sh"++# ─────────────────────────────────────────────────────────────────+start_test "plugin: health check"++pt_get /health+assert_json_eq "$RESULT" '.status' 'ok'+assert_result_exists ".version" "has version"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: navigate"++pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/form.html\"}"+assert_ok "navigate"+assert_result_exists ".tabId" "returns tabId"+assert_json_contains "$RESULT" '.title' 'Form'+TAB_ID=$(echo "$RESULT" | jq -r '.tabId')++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: snapshot with tabId"++pt_get "/snapshot?tabId=${TAB_ID}"+assert_ok "snapshot"+assert_result_exists ".title" "has title"+assert_result_exists ".url" "has url"+assert_json_length_gte "$RESULT" '.nodes' 1++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: action click"++pt_post /action -d "{\"tabId\":\"${TAB_ID}\",\"kind\":\"click\",\"selector\":\"#submit-btn\"}"+assert_ok "click action"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: action type"++pt_post /action -d "{\"tabId\":\"${TAB_ID}\",\"kind\":\"type\",\"selector\":\"#username\",\"text\":\"testuser\"}"+assert_ok "type action"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: action scroll"++pt_post /action -d "{\"tabId\":\"${TAB_ID}\",\"kind\":\"scroll\",\"direction\":\"down\"}"+assert_ok "scroll action"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: action press key"++pt_post /action -d "{\"tabId\":\"${TAB_ID}\",\"kind\":\"press\",\"key\":\"Escape\"}"+assert_ok "press key"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: action select"++pt_post /action -d "{\"tabId\":\"${TAB_ID}\",\"kind\":\"select\",\"selector\":\"#country\",\"value\":\"us\"}"+assert_ok "select action"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: screenshot"++RESPONSE=$(e2e_curl -s -w "\n%{http_code}" "${E2E_SERVER}/screenshot?tabId=${TAB_ID}")+STATUS=$(echo "$RESPONSE" | tail -n 1)++if [ "$STATUS" = "200" ]; then+ echo -e " ${GREEN}✓${NC} screenshot returned 200"+ ((ASSERTIONS_PASSED++)) || true+else+ echo -e " ${RED}✗${NC} screenshot returned $STATUS"+ ((ASSERTIONS_FAILED++)) || true+fi++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: wait for selector"++pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/buttons.html\"}"+TAB_ID=$(echo "$RESULT" | jq -r '.tabId')++pt_post /wait -d "{\"tabId\":\"${TAB_ID}\",\"selector\":\"button\"}"+assert_ok "wait for selector"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: wait for text"++pt_post /wait -d "{\"tabId\":\"${TAB_ID}\",\"text\":\"Click me\"}"+assert_ok "wait for text"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: tabs list"++pt_get /tabs+assert_ok "tabs list"+assert_json_length_gte "$RESULT" '.' 1 "has at least one tab"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: tab new"++pt_post /tab -d '{"action":"new"}'+assert_ok "new tab"+NEW_TAB=$(echo "$RESULT" | jq -r '.tabId')+assert_result_exists ".tabId" "returns tabId"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: tab close"++pt_post /tab -d "{\"action\":\"close\",\"tabId\":\"${NEW_TAB}\"}"+assert_ok "close tab"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: evaluate JavaScript"++pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/evaluate.html\"}"+TAB_ID=$(echo "$RESULT" | jq -r '.tabId')+sleep 1++pt_post /evaluate -d "{\"tabId\":\"${TAB_ID}\",\"expression\":\"document.title\"}"+assert_ok "evaluate"+assert_result_eq ".result" "Evaluate Test Page" "got title"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: text extraction"++pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/index.html\"}"+TAB_ID=$(echo "$RESULT" | jq -r '.tabId')++pt_get "/text?tabId=${TAB_ID}"+assert_ok "text extraction"+assert_result_exists ".text" "has text field"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: PDF generation"++RESPONSE=$(e2e_curl -s -w "\n%{http_code}" "${E2E_SERVER}/pdf?tabId=${TAB_ID}")+STATUS=$(echo "$RESPONSE" | tail -n 1)++if [ "$STATUS" = "200" ]; then+ echo -e " ${GREEN}✓${NC} pdf returned 200"+ ((ASSERTIONS_PASSED++)) || true+else+ echo -e " ${RED}✗${NC} pdf returned $STATUS"+ ((ASSERTIONS_FAILED++)) || true+fi++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: network log"++pt_get "/network?tabId=${TAB_ID}"+assert_ok "network log"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: instances list"++pt_get /instances+assert_ok "instances"++end_test++# ─────────────────────────────────────────────────────────────────+start_test "plugin: profiles list"++pt_get /profiles+assert_ok "profiles"++end_test