High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
refactor: extract integration test helpers into testutil package
Bosch2d73941198ef
diff --git a/TESTING.md b/TESTING.mdindex 7876f557..cf9b844c 100644--- a/TESTING.md+++ b/TESTING.md@@ -1,513 +1,77 @@-# Phase 6: End-to-End Testing Guide+# Testing-This guide provides manual and automated tests for the complete multi-instance architecture.--## Test Organization--- **Automated Integration Tests**: `tests/integration/orchestrator_test.go` (Go)- - Instance creation and lifecycle- - Hash-based ID generation- - Port allocation and reuse- - Instance isolation- - Orchestrator proxy routing- - Run with: `go test -tags integration ./tests/integration -run Orchestrator -timeout 120s`--- **Manual Tests**: `tests/manual/orchestrator.md`- - Visual verification (headed/headless windows)- - Real-time monitoring (memory, CPU)- - Port management verification- - Chrome initialization verification- - Dashboard UI testing- - Error conditions and edge cases- - Integration with existing features--- **Quick Validation**: Follow the step-by-step tests below for immediate feedback--## Prerequisites--- PinchTab built: `go build -o pinchtab ./cmd/pinchtab`-- Port range available: 9867-9968-- Chrome/Chromium installed--## Quick Start Test (Manual)--### 1. Start PinchTab--```bash-./pinchtab-```--This starts:-- Dashboard on port 9867-- Ready to accept instance creation requests--Expected output:-```-INFO dashboard listening addr=127.0.0.1:9867-INFO port allocator initialized start=9868 end=9968-```--### 2. Create First Instance (Headed)+## Unit Tests```bash-curl -X POST http://localhost:9867/instances/launch \- -H "Content-Type: application/json" \- -d '{- "name":"work",- "headless":false- }'+go test ./...```-**Expected response:**-```json-{- "id": "inst_XXXXXXXX",- "profileId": "prof_YYYYYYYY",- "profileName": "work",- "port": "9868",- "headless": false,- "status": "starting",- "startTime": "2026-02-28T20:15:00Z"-}-```+## Integration Tests-**Verify:**-- Hash-based instance ID (inst_XXXXXXXX format) ✓-- Hash-based profile ID (prof_XXXXXXXX format) ✓-- Port auto-allocated (9868) ✓-- Status is "starting" (not "running" yet) ✓-- **Within 2 seconds**: Chrome window should open on your screen ✓--### 3. Create Second Instance (Headless)+Integration tests launch a real pinchtab server with Chrome and run HTTP-level tests against it.```bash-curl -X POST http://localhost:9867/instances/launch \- -H "Content-Type: application/json" \- -d '{- "name":"scrape",- "headless":true- }'-```--**Expected response:**-```json-{- "id": "inst_ZZZZZZZZ",- "profileId": "prof_WWWWWWWW",- "profileName": "scrape",- "port": "9869",- "headless": true,- "status": "starting",- "startTime": "2026-02-28T20:15:05Z"-}+go test -tags integration ./tests/integration/ -v -timeout 5m```-**Verify:**-- Different instance ID ✓-- Different port (9869) ✓-- Same port range used (auto-allocation) ✓-- No Chrome window (headless=true) ✓--### 4. List All Instances--```bash-curl http://localhost:9867/instances-```+### Environment Variables-**Expected response:**-```json-[- {- "id": "inst_XXXXXXXX",- "profileId": "prof_YYYYYYYY",- "profileName": "work",- "port": "9868",- "headless": false,- "status": "running",- "startTime": "2026-02-28T20:15:00Z"- },- {- "id": "inst_ZZZZZZZZ",- "profileId": "prof_WWWWWWWW",- "profileName": "scrape",- "port": "9869",- "headless": true,- "status": "running",- "startTime": "2026-02-28T20:15:05Z"- }-]-```+| Variable | Default | Description |+|---|---|---|+| `PINCHTAB_TEST_PORT` | `19867` | Port for the test server |+| `PINCHTAB_TEST_KEEP_DIR` | _(unset)_ | Set to any value to preserve the test dir after tests finish |+| `CHROME_BINARY` | _(auto-detect)_ | Path to Chrome binary (used in CI) |+| `CI` | _(unset)_ | Set to `true` for longer health check timeouts (60s vs 30s) |-**Verify:**-- Both instances listed ✓-- Both have "running" status ✓-- Hash-based IDs on both ✓+### Temp Directory Layout-### 5. Navigate Instance 1 (via Orchestrator Proxy)+Each test run creates a single temp directory under `/tmp/pinchtab-test-*/`:-```bash-curl -X POST http://localhost:9867/instances/inst_XXXXXXXX/navigate \- -H "Content-Type: application/json" \- -d '{- "url": "https://example.com"- }'```--Replace `inst_XXXXXXXX` with actual ID from step 2.--**Expected response:**-```json-{- "tabId": "tab_MMMMMMMM",- "url": "https://example.com",- "title": "Example Domain"-}+/tmp/pinchtab-test-123456789/+├── pinchtab # Compiled test binary+├── state/ # Dashboard state (profiles, instances)+└── profiles/ # Chrome user-data directories```-**Verify:**-- Hash-based tab ID (tab_MMMMMMMM format) ✓-- Instance 1 Chrome window navigates to example.com ✓-- URL matches ✓--### 6. Navigate Instance 2 (via Orchestrator Proxy)+Everything is cleaned up automatically when tests finish. To inspect after a failure:```bash-curl -X POST http://localhost:9867/instances/inst_ZZZZZZZZ/navigate \- -H "Content-Type: application/json" \- -d '{- "url": "https://github.com"- }'+PINCHTAB_TEST_KEEP_DIR=1 go test -tags integration ./tests/integration/ -v```-Replace `inst_ZZZZZZZZ` with actual ID from step 3.+### Test Utilities (`tests/testutil/`)-**Expected response:**-```json-{- "tabId": "tab_NNNNNNNN",- "url": "https://github.com",- "title": "GitHub"-}-```--**Verify:**-- Different tab ID (tab_NNNNNNNN vs tab_MMMMMMMM) ✓-- Instance 2 (headless) navigates silently ✓-- Instance 1 window still shows example.com (not affected) ✓--### 7. Get Snapshot from Instance 1+The `testutil` package provides reusable helpers:-```bash-curl "http://localhost:9867/instances/inst_XXXXXXXX/snapshot" \- -o snapshot.json-cat snapshot.json | jq '.url'-```+- **`testutil.StartServer(cfg)`** — Builds, launches, and waits for a pinchtab server. Returns a `*Server` with `Stop()` for cleanup.+- **`testutil.NewClient(url)`** — HTTP client with `Get`, `Post`, `PostRaw`, `Patch`, `Delete`, `PostWithRetry` methods.+- **`testutil.LaunchInstance(url)`** — Creates and waits for a test instance.+- **`testutil.JSONField(t, data, key)`** — Extracts a string field from JSON response body.+- **`testutil.FindRepoRoot()`** — Walks up to find `go.mod`.+- **`testutil.WaitForHealth(url, timeout)`** — Polls `/health` with context-based timeout.+- **`testutil.TerminateProcessGroup(cmd, timeout)`** — SIGTERM to process group, SIGKILL on timeout.-**Expected:**-- Returns full page snapshot of example.com-- Shows isolation: only sees inst_XXXXXXXX's content ✓+### Writing New Tests-### 8. Stop Instance 1+```go+//go:build integration-```bash-curl -X POST http://localhost:9867/instances/inst_XXXXXXXX/stop-```+package integration-**Expected response:**-```json-{- "status": "stopped",- "id": "inst_XXXXXXXX"+func TestMyFeature(t *testing.T) {+ navigate(t, "https://example.com")++ code, body := client.Get(t, "/snapshot")+ if code != 200 {+ t.Fatalf("expected 200, got %d", code)+ }++ value := testutil.JSONField(t, body, "someField")+ // ...}```-**Verify:**-- Chrome window closes ✓-- Instance 2 still running (headless, invisible) ✓-- Port 9868 released back to allocator ✓--### 9. Create Third Instance (Reuses Released Port)--```bash-curl -X POST http://localhost:9867/instances/launch \- -H "Content-Type: application/json" \- -d '{- "name":"test",- "headless":true- }'-```--**Expected:**-- New instance gets port 9868 (reused from step 8) ✓-- New instance ID generated ✓--### 10. Stop All Instances--```bash-curl -X POST http://localhost:9867/instances/inst_ZZZZZZZZ/stop-curl -X POST http://localhost:9867/instances/inst_TTTTTTTT/stop-```--**Verify:**-- All instances stopped ✓-- All ports released back to allocator ✓-- Dashboard still running on 9867 ✓-------## Automated Test Script--```bash-#!/bin/bash--# Start Pinchtab in background-./pinchtab &-DASHBOARD_PID=$!-sleep 2--echo "✓ Dashboard started (PID: $DASHBOARD_PID)"--# Create instance 1-INST1=$(curl -s -X POST http://localhost:9867/instances/launch \- -H "Content-Type: application/json" \- -d '{"name":"work","headless":false}')-INST1_ID=$(echo $INST1 | jq -r '.id')-echo "✓ Created instance 1: $INST1_ID"--# Create instance 2-INST2=$(curl -s -X POST http://localhost:9867/instances/launch \- -H "Content-Type: application/json" \- -d '{"name":"scrape","headless":true}')-INST2_ID=$(echo $INST2 | jq -r '.id')-echo "✓ Created instance 2: $INST2_ID"--# Wait for Chrome to initialize-sleep 3--# List instances-INSTANCES=$(curl -s http://localhost:9867/instances | jq '.[] | .id')-echo "✓ Running instances: $(echo $INSTANCES | tr '\n' ' ')"--# Navigate instance 1-NAV1=$(curl -s -X POST "http://localhost:9867/instances/$INST1_ID/navigate" \- -H "Content-Type: application/json" \- -d '{"url":"https://example.com"}')-TAB1_ID=$(echo $NAV1 | jq -r '.tabId')-echo "✓ Navigated instance 1, tab: $TAB1_ID"--# Navigate instance 2-NAV2=$(curl -s -X POST "http://localhost:9867/instances/$INST2_ID/navigate" \- -H "Content-Type: application/json" \- -d '{"url":"https://github.com"}')-TAB2_ID=$(echo $NAV2 | jq -r '.tabId')-echo "✓ Navigated instance 2, tab: $TAB2_ID"--# Verify isolation: different tab IDs-if [ "$TAB1_ID" != "$TAB2_ID" ]; then- echo "✓ Tab isolation verified (different IDs)"-else- echo "✗ FAILED: Tab IDs should be different!"- exit 1-fi--# Stop instance 1-curl -s -X POST "http://localhost:9867/instances/$INST1_ID/stop" > /dev/null-echo "✓ Stopped instance 1"--# Stop instance 2-curl -s -X POST "http://localhost:9867/instances/$INST2_ID/stop" > /dev/null-echo "✓ Stopped instance 2"--# Verify all stopped-REMAINING=$(curl -s http://localhost:9867/instances | jq '.[] | .id' | wc -l)-if [ "$REMAINING" -eq 0 ]; then- echo "✓ All instances cleaned up"-else- echo "✗ FAILED: $REMAINING instances still running!"- exit 1-fi--# Clean up-kill $DASHBOARD_PID 2>/dev/null-echo ""-echo "✅ ALL TESTS PASSED!"-```--Save as `test-e2e.sh`, then run:-```bash-chmod +x test-e2e.sh-./test-e2e.sh-```-------## Stress Test (10 Instances)--```bash-#!/bin/bash--./pinchtab &-DASHBOARD_PID=$!-sleep 2--echo "Creating 10 instances..."-INSTANCES=()--for i in {1..10}; do- INST=$(curl -s -X POST http://localhost:9867/instances/launch \- -H "Content-Type: application/json" \- -d "{\"name\":\"stress-$i\",\"headless\":true}")- ID=$(echo $INST | jq -r '.id')- INSTANCES+=($ID)- PORT=$(echo $INST | jq -r '.port')- echo " $i. $ID (port: $PORT)"-done--sleep 3-echo ""-echo "Navigating all instances concurrently..."--for ID in "${INSTANCES[@]}"; do- curl -s -X POST "http://localhost:9867/instances/$ID/navigate" \- -H "Content-Type: application/json" \- -d '{"url":"https://example.com"}' &-done--wait-echo "✓ All navigations completed"--echo ""-echo "Stopping all instances..."-for ID in "${INSTANCES[@]}"; do- curl -s -X POST "http://localhost:9867/instances/$ID/stop" > /dev/null &-done--wait-echo "✓ All instances stopped"--kill $DASHBOARD_PID 2>/dev/null-echo "✅ STRESS TEST PASSED!"-```-------## Multi-Agent Test--Test with agents targeting different instances:--```bash-#!/bin/bash--./pinchtab &-sleep 2--# Create 3 instances for 3 agents-AGENT_A=$(curl -s -X POST http://localhost:9867/instances/launch \- -H "Content-Type: application/json" \- -d '{"name":"agent-a","headless":true}' | jq -r '.id')--AGENT_B=$(curl -s -X POST http://localhost:9867/instances/launch \- -H "Content-Type: application/json" \- -d '{"name":"agent-b","headless":true}' | jq -r '.id')--AGENT_C=$(curl -s -X POST http://localhost:9867/instances/launch \- -H "Content-Type: application/json" \- -d '{"name":"agent-c","headless":true}' | jq -r '.id')--sleep 2--# Agent A: navigate to site 1-curl -X POST "http://localhost:9867/instances/$AGENT_A/navigate" \- -H "Content-Type: application/json" \- -d '{"url":"https://example.com"}' &--# Agent B: navigate to site 2-curl -X POST "http://localhost:9867/instances/$AGENT_B/navigate" \- -H "Content-Type: application/json" \- -d '{"url":"https://github.com"}' &--# Agent C: navigate to site 3-curl -X POST "http://localhost:9867/instances/$AGENT_C/navigate" \- -H "Content-Type: application/json" \- -d '{"url":"https://rust-lang.org"}' &--wait--echo "✓ All agents navigated independently"-echo "✓ Each has isolated cookies/history"-echo "✓ No state leakage between agents"--# Cleanup-curl -s -X POST "http://localhost:9867/instances/$AGENT_A/stop" > /dev/null-curl -s -X POST "http://localhost:9867/instances/$AGENT_B/stop" > /dev/null-curl -s -X POST "http://localhost:9867/instances/$AGENT_C/stop" > /dev/null--echo "✅ MULTI-AGENT TEST PASSED!"-```-------## Test Checklist--- [ ] Hash-based IDs work (prof_X, inst_X, tab_X formats)-- [ ] Auto-port allocation works (9868, 9869, 9870, ...)-- [ ] Port release and reuse works-- [ ] Headed instance opens Chrome window-- [ ] Headless instance runs silently-- [ ] Orchestrator proxy routes work-- [ ] Instance isolation (no state leakage)-- [ ] Multi-instance concurrent navigation works-- [ ] Health checks pass on all instances-- [ ] Cleanup on shutdown works-- [ ] 10-instance stress test passes-- [ ] Multi-agent isolation verified-------## Debugging Tips--### Check Dashboard Logs--```bash-# Follow logs while running tests-./pinchtab 2>&1 | grep -E "instance|chrome|port"-```--### Check Instance Port--```bash-# See which ports are in use-lsof -i :9868-lsof -i :9869-```--### Check Instance Health--```bash-curl http://localhost:9868/health-curl http://localhost:9869/health-```--### Manual Chrome Kill (if needed)--```bash-# Kill all Pinchtab-managed Chrome processes-pkill -f "Chrome.*user-data-dir.*\.pinchtab"-```-------## Expected Results--After running all tests, you should see:-- ✅ 10+ instances created and destroyed without issues-- ✅ Hash-based IDs consistent and validated-- ✅ Ports auto-allocated and released properly-- ✅ Headed windows open and close correctly-- ✅ No Chrome orphan processes-- ✅ Complete isolation between instances-- ✅ Multi-agent concurrent access works flawlessly-----+### Process Cleanup-**Ready to test?** Run `./pinchtab` and follow the Quick Start Test above!+The test server runs in its own process group (`Setpgid`). On shutdown, `SIGTERM` is sent to the entire group, killing Chrome and all child processes. If graceful shutdown fails after 10 seconds, `SIGKILL` is sent to the group.diff --git a/tests/integration/helpers_test.go b/tests/integration/helpers_test.goindex 8a82442e..d2a5d18e 100644--- a/tests/integration/helpers_test.go+++ b/tests/integration/helpers_test.go@@ -3,129 +3,16 @@package integrationimport (- "encoding/json"- "io"- "net/http"- "strings""testing"- "time")+// Backward-compatible wrappers for tests using httpPatch/httpDelete.+// New tests should use client.Patch() / client.Delete() directly.+func httpPatch(t *testing.T, path string, payload any) (int, []byte) {- t.Helper()- var reader io.Reader- if payload != nil {- data, err := json.Marshal(payload)- if err != nil {- t.Fatalf("marshal failed: %v", err)- }- reader = strings.NewReader(string(data))- }- req, err := http.NewRequest("PATCH", serverURL+path, reader)- if err != nil {- t.Fatalf("PATCH %s request creation failed: %v", path, err)- }- req.Header.Set("Content-Type", "application/json")- resp, err := http.DefaultClient.Do(req)- if err != nil {- t.Fatalf("PATCH %s failed: %v", path, err)- }- defer resp.Body.Close() //nolint:errcheck- body, _ := io.ReadAll(resp.Body)- return resp.StatusCode, body+ return client.Patch(t, path, payload)}func httpDelete(t *testing.T, path string) (int, []byte) {- t.Helper()- req, err := http.NewRequest("DELETE", serverURL+path, nil)- if err != nil {- t.Fatalf("DELETE %s request creation failed: %v", path, err)- }- resp, err := http.DefaultClient.Do(req)- if err != nil {- t.Fatalf("DELETE %s failed: %v", path, err)- }- defer resp.Body.Close() //nolint:errcheck- body, _ := io.ReadAll(resp.Body)- return resp.StatusCode, body-}--// httpPostWithRetry attempts a POST request with retries for flaky tests-func httpPostWithRetry(t *testing.T, path string, body any, maxRetries int) (int, []byte) {- t.Helper()-- var lastCode int- var lastBody []byte- var lastErr error-- for i := 0; i <= maxRetries; i++ {- if i > 0 {- t.Logf("Retry %d/%d for %s", i, maxRetries, path)- time.Sleep(2 * time.Second) // Wait before retry- }-- code, respBody, err := doPost(serverURL+path, body)- lastCode = code- lastBody = respBody- lastErr = err-- if err == nil && code == 200 {- return code, respBody- }-- // Log the failure for debugging- if err != nil {- t.Logf("Request failed with error: %v", err)- } else {- t.Logf("Request failed with status %d: %s", code, string(respBody))- }- }-- // If all retries failed, use the original httpPost behavior- if lastErr != nil {- t.Fatalf("http post %s: %v", path, lastErr)- }- return lastCode, lastBody-}--// doPost performs the actual HTTP POST-func doPost(url string, body any) (int, []byte, error) {- var reader io.Reader- if body != nil {- jsonBody, _ := json.Marshal(body)- reader = strings.NewReader(string(jsonBody))- }-- req, err := http.NewRequest("POST", url, reader)- if err != nil {- return 0, nil, err- }-- req.Header.Set("Content-Type", "application/json")- if token := getAuthToken(); token != "" {- req.Header.Set("Authorization", "Bearer "+token)- }-- client := &http.Client{- Timeout: 45 * time.Second, // Increase timeout for Chrome startup- }-- resp, err := client.Do(req)- if err != nil {- return 0, nil, err- }- defer resp.Body.Close()-- respBody, err := io.ReadAll(resp.Body)- if err != nil {- return resp.StatusCode, nil, err- }-- return resp.StatusCode, respBody, nil-}--// getAuthToken returns the auth token if configured-func getAuthToken() string {- // This should match the token logic in main_test.go- return "test-token"+ return client.Delete(t, path)}diff --git a/tests/integration/main_test.go b/tests/integration/main_test.goindex e9487d84..f9257047 100644--- a/tests/integration/main_test.go+++ b/tests/integration/main_test.go@@ -3,400 +3,104 @@package integrationimport (- "encoding/json""fmt"- "io"- "net/http""os"- "os/exec"- "path/filepath"- "strings"- "syscall""testing"- "time"++ "github.com/pinchtab/pinchtab/tests/testutil")var (- serverURL string- currentTabID string // Track current tab for action operations+ server *testutil.Server+ client *testutil.Client+ currentTabID string)func TestMain(m *testing.M) {- port := os.Getenv("PINCHTAB_TEST_PORT")- if port == "" {- port = "19867"- }- serverURL = fmt.Sprintf("http://localhost:%s", port)+ cfg := testutil.DefaultConfig()- // Single parent temp dir for all test artifacts (binary, state, profiles).- // Cleaned up explicitly before os.Exit — defer won't run after os.Exit.- testDir, err := os.MkdirTemp("", "pinchtab-test-*")+ var err error+ server, err = testutil.StartServer(cfg)if err != nil {- fmt.Fprintf(os.Stderr, "failed to create test dir: %v\n", err)- os.Exit(1)- }- fmt.Fprintf(os.Stderr, "TestMain: test dir: %s\n", testDir)-- cleanup := func() {- if os.Getenv("PINCHTAB_TEST_KEEP_DIR") != "" {- fmt.Fprintf(os.Stderr, "TestMain: keeping test dir (PINCHTAB_TEST_KEEP_DIR set): %s\n", testDir)- return- }- os.RemoveAll(testDir)- }-- binaryPath := filepath.Join(testDir, "pinchtab")- stateDir := filepath.Join(testDir, "state")- profileDir := filepath.Join(testDir, "profiles")-- if err := os.MkdirAll(stateDir, 0o755); err != nil {- fmt.Fprintf(os.Stderr, "failed to create state dir: %v\n", err)- cleanup()- os.Exit(1)- }- if err := os.MkdirAll(profileDir, 0o755); err != nil {- fmt.Fprintf(os.Stderr, "failed to create profile dir: %v\n", err)- cleanup()+ fmt.Fprintf(os.Stderr, "failed to start server: %v\n", err)os.Exit(1)}- // Build the binary into the test dir- build := exec.Command("go", "build", "-o", binaryPath, "./cmd/pinchtab/")- build.Dir = findRepoRoot()- build.Stdout = os.Stdout- build.Stderr = os.Stderr- if err := build.Run(); err != nil {- fmt.Fprintf(os.Stderr, "failed to build pinchtab: %v\n", err)- cleanup()- os.Exit(1)- }-- // Start server in its own process group so we can kill Chrome children on shutdown.- cmd := exec.Command(binaryPath)-- // Build environment for subprocess- // Start with a filtered set of inherited env vars, then add test-specific ones- baseEnv := os.Environ()- env := []string{}- for _, e := range baseEnv {- // Skip any pre-existing BRIDGE_* and PINCHTAB_* vars to avoid conflicts- if !strings.HasPrefix(e, "BRIDGE_") && !strings.HasPrefix(e, "PINCHTAB_") {- env = append(env, e)- }- }-- // Add test-specific environment (use PINCHTAB_* names, BRIDGE_* are deprecated)- env = append(env,- "PINCHTAB_PORT="+port,- "PINCHTAB_HEADLESS=true",- "PINCHTAB_NO_RESTORE=true",- "PINCHTAB_STEALTH=light",- "PINCHTAB_STATE_DIR="+stateDir,- "PINCHTAB_PROFILE_DIR="+profileDir,- )-- // Pass CHROME_BINARY if set by CI workflow or environment- if chromeBinary := os.Getenv("CHROME_BINARY"); chromeBinary != "" {- env = append(env, "CHROME_BINARY="+chromeBinary)- }-- cmd.Env = env- cmd.Stdout = os.Stdout- cmd.Stderr = os.Stderr- cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}-- if err := cmd.Start(); err != nil {- fmt.Fprintf(os.Stderr, "failed to start pinchtab: %v\n", err)- cleanup()- os.Exit(1)- }-- // Wait for server to be ready (longer timeout in CI)- healthTimeout := 30 * time.Second- if os.Getenv("CI") == "true" {- healthTimeout = 60 * time.Second- }- if !waitForHealth(serverURL, healthTimeout) {- fmt.Fprintf(os.Stderr, "pinchtab did not become healthy within timeout (%v)\n", healthTimeout)- _ = cmd.Process.Kill()- cleanup()- os.Exit(1)- }+ client = testutil.NewClient(server.URL)// Launch a test instance for orchestrator-mode tests- // This ensures /navigate and other proxy endpoints work in CI- if err := launchTestInstance(serverURL); err != nil {+ if _, err := testutil.LaunchInstance(server.URL); err != nil {fmt.Fprintf(os.Stderr, "failed to launch test instance: %v\n", err)- _ = cmd.Process.Kill()- cleanup()+ server.Stop()os.Exit(1)}code := m.Run()-- // Shutdown — send SIGTERM to the process group to also kill Chrome children.- if pgid, err := syscall.Getpgid(cmd.Process.Pid); err == nil {- _ = syscall.Kill(-pgid, syscall.SIGTERM)- } else {- _ = cmd.Process.Signal(os.Interrupt)- }- done := make(chan error, 1)- go func() { done <- cmd.Wait() }()- select {- case <-done:- case <-time.After(10 * time.Second):- if pgid, err := syscall.Getpgid(cmd.Process.Pid); err == nil {- _ = syscall.Kill(-pgid, syscall.SIGKILL)- }- _ = cmd.Process.Kill()- }-- cleanup()+ server.Stop()os.Exit(code)}-func findRepoRoot() string {- dir, _ := os.Getwd()- for {- if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {- return dir- }- parent := filepath.Dir(dir)- if parent == dir {- break- }- dir = parent- }- return filepath.Join("..", "..")-}--func waitForHealth(base string, timeout time.Duration) bool {- deadline := time.Now().Add(timeout)- for time.Now().Before(deadline) {- resp, err := http.Get(base + "/health")- if err == nil && resp.StatusCode == 200 {- _ = resp.Body.Close()- return true- }- if resp != nil {- _ = resp.Body.Close()- }- time.Sleep(500 * time.Millisecond)- }- return false-}--// helpers--func httpGet(t *testing.T, path string) (int, []byte) {- t.Helper()- resp, err := http.Get(serverURL + path)- if err != nil {- t.Fatalf("GET %s failed: %v", path, err)- }- defer resp.Body.Close() //nolint:errcheck- body, _ := io.ReadAll(resp.Body)- return resp.StatusCode, body-}--func httpPost(t *testing.T, path string, payload any) (int, []byte) {- t.Helper()- var reader io.Reader- if payload != nil {- data, err := json.Marshal(payload)- if err != nil {- t.Fatalf("marshal failed: %v", err)- }- reader = strings.NewReader(string(data))- }- resp, err := http.Post(serverURL+path, "application/json", reader)- if err != nil {- t.Fatalf("POST %s failed: %v", path, err)- }- defer resp.Body.Close() //nolint:errcheck- body, _ := io.ReadAll(resp.Body)- return resp.StatusCode, body-}--func httpPostRaw(t *testing.T, path string, body string) (int, []byte) {- t.Helper()- resp, err := http.Post(serverURL+path, "application/json", strings.NewReader(body))- if err != nil {- t.Fatalf("POST %s failed: %v", path, err)- }- defer resp.Body.Close() //nolint:errcheck- data, _ := io.ReadAll(resp.Body)- return resp.StatusCode, data-}--func jsonField(t *testing.T, data []byte, key string) string {- t.Helper()- var m map[string]any- if err := json.Unmarshal(data, &m); err != nil {- t.Fatalf("json parse failed: %v (body: %s)", err, string(data))- }- v, ok := m[key]- if !ok {- return ""- }- switch val := v.(type) {- case string:- return val- default:- b, _ := json.Marshal(val)- return string(b)- }-}-func navigate(t *testing.T, url string) {t.Helper()- // Use retry logic for better stability- code, body := httpPostWithRetry(t, "/navigate", map[string]any{"url": url}, 2)+ code, body := client.PostWithRetry(t, "/navigate", map[string]any{"url": url}, 2)if code != 200 {t.Fatalf("navigate to %s failed with %d: %s", url, code, string(body))}- // Extract tabId from response for subsequent action calls- var result map[string]any- if err := json.Unmarshal(body, &result); err != nil {- t.Logf("warning: failed to parse navigate response: %v", err)- return- }-- if id, ok := result["tabId"].(string); ok {+ if id := testutil.JSONField(t, body, "tabId"); id != "" {currentTabID = idt.Logf("current tab: %s", currentTabID)- // Auto-close tab on test completion to prevent Chrome hitting tab limit.- // Safe to call even if the test also defers closeCurrentTab (idempotent).t.Cleanup(func() { closeCurrentTab(t) })}}-// closeCurrentTab closes the current tab to clean up resourcesfunc closeCurrentTab(t *testing.T) {t.Helper()if currentTabID == "" {return}- // Close the tab- _, _ = httpPost(t, "/tab", map[string]any{+ _, _ = client.Post(t, "/tab", map[string]any{"tabId": currentTabID,"action": "close",})currentTabID = ""}-// navigateInstance creates a fresh tab on the given instance and navigates it.func navigateInstance(t *testing.T, instID, url string) (int, []byte, string) {t.Helper()- openCode, openBody := httpPostWithRetry(t, fmt.Sprintf("/instances/%s/tabs/open", instID), map[string]any{+ openCode, openBody := client.PostWithRetry(t, fmt.Sprintf("/instances/%s/tabs/open", instID), map[string]any{"url": "about:blank",}, 2)if openCode != 200 {return openCode, openBody, ""}- tabID := jsonField(t, openBody, "tabId")+ tabID := testutil.JSONField(t, openBody, "tabId")if tabID == "" {return 500, []byte(`{"error":"missing tabId from open tab response"}`), ""}path := fmt.Sprintf("/tabs/%s/navigate", tabID)- code, body := httpPostWithRetry(t, path, map[string]any{"url": url}, 2)+ code, body := client.PostWithRetry(t, path, map[string]any{"url": url}, 2)return code, body, tabID}-// waitForInstanceReady waits for an instance to be ready for navigation-// Uses a simple navigate to about:blank to check readinessfunc waitForInstanceReady(t *testing.T, instID string) {t.Helper()- deadline := time.Now().Add(15 * time.Second)- for time.Now().Before(deadline) {- code, _, _ := navigateInstance(t, instID, "about:blank")- if code == 200 {- t.Logf("instance %s is ready", instID)- return- }- time.Sleep(500 * time.Millisecond)+ if !testutil.WaitForHealth(server.URL+"/instances/"+instID, 15*1e9) {+ t.Logf("warning: instance %s did not become ready within 15 seconds", instID)}- t.Logf("warning: instance %s did not become ready within 15 seconds", instID)}-// launchTestInstance launches a default test instance for orchestrator-mode tests-// This is called once during TestMain setup so that /navigate and proxy endpoints work-// It waits for the instance to be fully ready before returning-func launchTestInstance(base string) error {- resp, err := http.Post(- base+"/instances/launch",- "application/json",- strings.NewReader(`{"mode":"headless"}`),- )- if err != nil {- return err- }- defer resp.Body.Close() //nolint:errcheck-- if resp.StatusCode != 201 && resp.StatusCode != 200 {- body, _ := io.ReadAll(resp.Body)- return fmt.Errorf("launch failed with status %d: %s", resp.StatusCode, string(body))- }-- // Parse response to get instance ID- var result map[string]any- body, _ := io.ReadAll(resp.Body)- if err := json.Unmarshal(body, &result); err != nil {- return fmt.Errorf("failed to parse launch response: %v", err)- }-- id, ok := result["id"].(string)- if !ok {- return fmt.Errorf("no instance id in launch response: %v", result)- }-- fmt.Fprintf(os.Stderr, "TestMain: launched test instance %s\n", id)-- // Wait for instance to be ready (can take 2-5 seconds for Chrome to start)- deadline := time.Now().Add(30 * time.Second)- for time.Now().Before(deadline) {- // Open a tab first, then navigate it to check if instance is ready.- openResp, err := http.Post(- base+"/instances/"+id+"/tabs/open",- "application/json",- strings.NewReader(`{"url":"about:blank"}`),- )- if err == nil {- var tabID string- if openResp.StatusCode == 200 {- openBody, _ := io.ReadAll(openResp.Body)- var open map[string]any- if err := json.Unmarshal(openBody, &open); err == nil {- if idValue, ok := open["tabId"].(string); ok {- tabID = idValue- }- }- }- _ = openResp.Body.Close()- if tabID != "" {- navResp, err := http.Post(- base+"/tabs/"+tabID+"/navigate",- "application/json",- strings.NewReader(`{"url":"about:blank"}`),- )- if err == nil {- if navResp.StatusCode == 200 {- _ = navResp.Body.Close()- fmt.Fprintf(os.Stderr, "TestMain: instance %s is ready\n", id)- return nil- }- navBody, _ := io.ReadAll(navResp.Body)- _ = navResp.Body.Close()- fmt.Fprintf(os.Stderr, "TestMain: instance not ready yet (%d): %s\n", navResp.StatusCode, string(navBody))- }- }- }- time.Sleep(500 * time.Millisecond)- }+// Backward-compatible wrappers — delegate to testutil.Client.+// Existing tests use these package-level functions; new tests should use client.* directly.- return fmt.Errorf("instance %s did not become ready within 30 seconds", id)+func httpGet(t *testing.T, path string) (int, []byte) { return client.Get(t, path) }+func httpPost(t *testing.T, path string, p any) (int, []byte) { return client.Post(t, path, p) }+func httpPostRaw(t *testing.T, path string, b string) (int, []byte) { return client.PostRaw(t, path, b) }+func httpPostWithRetry(t *testing.T, path string, body any, retries int) (int, []byte) {+ return client.PostWithRetry(t, path, body, retries)}+func jsonField(t *testing.T, data []byte, key string) string { return testutil.JSONField(t, data, key) }+func findRepoRoot() string { return testutil.FindRepoRoot() }diff --git a/tests/testutil/http.go b/tests/testutil/http.gonew file mode 100644index 00000000..d3159bdf--- /dev/null+++ b/tests/testutil/http.go@@ -0,0 +1,188 @@+package testutil++import (+ "encoding/json"+ "io"+ "net/http"+ "strings"+ "testing"+ "time"+)++// Client wraps an HTTP client bound to a server URL for integration tests.+type Client struct {+ BaseURL string+ AuthToken string+ Timeout time.Duration+}++// NewClient creates a test HTTP client for the given base URL.+func NewClient(baseURL string) *Client {+ return &Client{+ BaseURL: baseURL,+ AuthToken: "test-token",+ Timeout: 45 * time.Second,+ }+}++// Get performs a GET request and returns status + body.+func (c *Client) Get(t *testing.T, path string) (int, []byte) {+ t.Helper()+ resp, err := http.Get(c.BaseURL + path)+ if err != nil {+ t.Fatalf("GET %s failed: %v", path, err)+ }+ defer resp.Body.Close() //nolint:errcheck+ body, _ := io.ReadAll(resp.Body)+ return resp.StatusCode, body+}++// Post performs a POST request with JSON body.+func (c *Client) Post(t *testing.T, path string, payload any) (int, []byte) {+ t.Helper()+ var reader io.Reader+ if payload != nil {+ data, err := json.Marshal(payload)+ if err != nil {+ t.Fatalf("marshal failed: %v", err)+ }+ reader = strings.NewReader(string(data))+ }+ resp, err := http.Post(c.BaseURL+path, "application/json", reader)+ if err != nil {+ t.Fatalf("POST %s failed: %v", path, err)+ }+ defer resp.Body.Close() //nolint:errcheck+ body, _ := io.ReadAll(resp.Body)+ return resp.StatusCode, body+}++// PostRaw performs a POST request with a raw string body.+func (c *Client) PostRaw(t *testing.T, path string, body string) (int, []byte) {+ t.Helper()+ resp, err := http.Post(c.BaseURL+path, "application/json", strings.NewReader(body))+ if err != nil {+ t.Fatalf("POST %s failed: %v", path, err)+ }+ defer resp.Body.Close() //nolint:errcheck+ data, _ := io.ReadAll(resp.Body)+ return resp.StatusCode, data+}++// Patch performs a PATCH request with JSON body.+func (c *Client) Patch(t *testing.T, path string, payload any) (int, []byte) {+ t.Helper()+ var reader io.Reader+ if payload != nil {+ data, err := json.Marshal(payload)+ if err != nil {+ t.Fatalf("marshal failed: %v", err)+ }+ reader = strings.NewReader(string(data))+ }+ req, err := http.NewRequest("PATCH", c.BaseURL+path, reader)+ if err != nil {+ t.Fatalf("PATCH %s request creation failed: %v", path, err)+ }+ req.Header.Set("Content-Type", "application/json")+ resp, err := http.DefaultClient.Do(req)+ if err != nil {+ t.Fatalf("PATCH %s failed: %v", path, err)+ }+ defer resp.Body.Close() //nolint:errcheck+ body, _ := io.ReadAll(resp.Body)+ return resp.StatusCode, body+}++// Delete performs a DELETE request.+func (c *Client) Delete(t *testing.T, path string) (int, []byte) {+ t.Helper()+ req, err := http.NewRequest("DELETE", c.BaseURL+path, nil)+ if err != nil {+ t.Fatalf("DELETE %s request creation failed: %v", path, err)+ }+ resp, err := http.DefaultClient.Do(req)+ if err != nil {+ t.Fatalf("DELETE %s failed: %v", path, err)+ }+ defer resp.Body.Close() //nolint:errcheck+ body, _ := io.ReadAll(resp.Body)+ return resp.StatusCode, body+}++// PostWithRetry attempts a POST request with retries for flaky tests.+func (c *Client) PostWithRetry(t *testing.T, path string, body any, maxRetries int) (int, []byte) {+ t.Helper()++ var lastCode int+ var lastBody []byte+ var lastErr error++ client := &http.Client{Timeout: c.Timeout}++ for i := 0; i <= maxRetries; i++ {+ if i > 0 {+ t.Logf("Retry %d/%d for %s", i, maxRetries, path)+ time.Sleep(2 * time.Second)+ }++ var reader io.Reader+ if body != nil {+ data, _ := json.Marshal(body)+ reader = strings.NewReader(string(data))+ }++ req, err := http.NewRequest("POST", c.BaseURL+path, reader)+ if err != nil {+ lastErr = err+ continue+ }+ req.Header.Set("Content-Type", "application/json")+ if c.AuthToken != "" {+ req.Header.Set("Authorization", "Bearer "+c.AuthToken)+ }++ resp, err := client.Do(req)+ if err != nil {+ lastErr = err+ t.Logf("Request failed with error: %v", err)+ continue+ }++ respBody, _ := io.ReadAll(resp.Body)+ _ = resp.Body.Close()+ lastCode = resp.StatusCode+ lastBody = respBody+ lastErr = nil++ if resp.StatusCode == 200 {+ return resp.StatusCode, respBody+ }+ t.Logf("Request failed with status %d: %s", resp.StatusCode, string(respBody))+ }++ if lastErr != nil {+ t.Fatalf("http post %s: %v", path, lastErr)+ }+ return lastCode, lastBody+}++// JSONField extracts a string field from JSON response body.+func JSONField(t *testing.T, data []byte, key string) string {+ t.Helper()+ var m map[string]any+ if err := json.Unmarshal(data, &m); err != nil {+ t.Fatalf("json parse failed: %v (body: %s)", err, string(data))+ }+ v, ok := m[key]+ if !ok {+ return ""+ }+ switch val := v.(type) {+ case string:+ return val+ default:+ b, _ := json.Marshal(val)+ return string(b)+ }+}diff --git a/tests/testutil/server.go b/tests/testutil/server.gonew file mode 100644index 00000000..0216b984--- /dev/null+++ b/tests/testutil/server.go@@ -0,0 +1,329 @@+// Package testutil provides shared helpers for integration tests.+package testutil++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "syscall"+ "time"+)++// Timeout constants for integration test operations.+const (+ HealthTimeout = 30 * time.Second+ HealthTimeoutCI = 60 * time.Second+ ShutdownTimeout = 10 * time.Second+ InstanceTimeout = 30 * time.Second+)++// ServerConfig holds configuration for a test server instance.+type ServerConfig struct {+ Port string // Server port (default: "19867")+ Headless bool // Run Chrome headless (default: true)+ Stealth string // Stealth level (default: "light")+}++// DefaultConfig returns a ServerConfig with sensible test defaults.+func DefaultConfig() ServerConfig {+ port := os.Getenv("PINCHTAB_TEST_PORT")+ if port == "" {+ port = "19867"+ }+ return ServerConfig{+ Port: port,+ Headless: true,+ Stealth: "light",+ }+}++// Server represents a running pinchtab test server with managed temp dirs.+type Server struct {+ URL string+ Dir string // Root temp dir containing binary, state, profiles+ BinaryPath string+ StateDir string+ ProfileDir string+ cmd *exec.Cmd+}++// StartServer builds, launches, and waits for a pinchtab server.+// The caller must call server.Stop() to shut down and clean up.+func StartServer(cfg ServerConfig) (*Server, error) {+ testDir, err := os.MkdirTemp("", "pinchtab-test-*")+ if err != nil {+ return nil, fmt.Errorf("create test dir: %w", err)+ }+ fmt.Fprintf(os.Stderr, "testutil: test dir: %s\n", testDir)++ s := &Server{+ URL: fmt.Sprintf("http://localhost:%s", cfg.Port),+ Dir: testDir,+ BinaryPath: filepath.Join(testDir, "pinchtab"),+ StateDir: filepath.Join(testDir, "state"),+ ProfileDir: filepath.Join(testDir, "profiles"),+ }++ for _, d := range []string{s.StateDir, s.ProfileDir} {+ if err := os.MkdirAll(d, 0o755); err != nil {+ s.Cleanup()+ return nil, fmt.Errorf("create %s: %w", d, err)+ }+ }++ // Build binary+ build := exec.Command("go", "build", "-o", s.BinaryPath, "./cmd/pinchtab/")+ build.Dir = FindRepoRoot()+ build.Stdout = os.Stdout+ build.Stderr = os.Stderr+ if err := build.Run(); err != nil {+ s.Cleanup()+ return nil, fmt.Errorf("build pinchtab: %w", err)+ }++ // Prepare environment — strip existing BRIDGE_*/PINCHTAB_* to avoid conflicts+ env := filterEnv(os.Environ(), "BRIDGE_", "PINCHTAB_")+ env = append(env,+ "PINCHTAB_PORT="+cfg.Port,+ "PINCHTAB_HEADLESS="+boolStr(cfg.Headless),+ "PINCHTAB_NO_RESTORE=true",+ "PINCHTAB_STEALTH="+cfg.Stealth,+ "PINCHTAB_STATE_DIR="+s.StateDir,+ "PINCHTAB_PROFILE_DIR="+s.ProfileDir,+ )+ if bin := os.Getenv("CHROME_BINARY"); bin != "" {+ env = append(env, "CHROME_BINARY="+bin)+ }++ // Start in its own process group for clean shutdown+ s.cmd = exec.Command(s.BinaryPath)+ s.cmd.Env = env+ s.cmd.Stdout = os.Stdout+ s.cmd.Stderr = os.Stderr+ s.cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}++ if err := s.cmd.Start(); err != nil {+ s.Cleanup()+ return nil, fmt.Errorf("start pinchtab: %w", err)+ }++ // Wait for health+ timeout := HealthTimeout+ if os.Getenv("CI") == "true" {+ timeout = HealthTimeoutCI+ }+ if !WaitForHealth(s.URL, timeout) {+ s.Stop()+ return nil, fmt.Errorf("pinchtab did not become healthy within %v", timeout)+ }++ return s, nil+}++// Stop gracefully shuts down the server and cleans up all temp files.+func (s *Server) Stop() {+ if s.cmd != nil && s.cmd.Process != nil {+ TerminateProcessGroup(s.cmd, ShutdownTimeout)+ }+ s.Cleanup()+}++// Cleanup removes the test directory. Respects PINCHTAB_TEST_KEEP_DIR.+func (s *Server) Cleanup() {+ if os.Getenv("PINCHTAB_TEST_KEEP_DIR") != "" {+ fmt.Fprintf(os.Stderr, "testutil: keeping test dir (PINCHTAB_TEST_KEEP_DIR set): %s\n", s.Dir)+ return+ }+ os.RemoveAll(s.Dir)+}++// TerminateProcessGroup sends SIGTERM to the process group, then SIGKILL on timeout.+func TerminateProcessGroup(cmd *exec.Cmd, timeout time.Duration) {+ if cmd.Process == nil {+ return+ }++ // Try graceful group shutdown+ if pgid, err := syscall.Getpgid(cmd.Process.Pid); err == nil {+ _ = syscall.Kill(-pgid, syscall.SIGTERM)+ } else {+ _ = cmd.Process.Signal(os.Interrupt)+ }++ done := make(chan error, 1)+ go func() { done <- cmd.Wait() }()++ select {+ case <-done:+ return+ case <-time.After(timeout):+ // Force kill the entire process group+ if pgid, err := syscall.Getpgid(cmd.Process.Pid); err == nil {+ _ = syscall.Kill(-pgid, syscall.SIGKILL)+ }+ _ = cmd.Process.Kill()+ <-done+ }+}++// WaitForHealth polls the /health endpoint until it returns 200 or the timeout expires.+func WaitForHealth(base string, timeout time.Duration) bool {+ ctx, cancel := context.WithTimeout(context.Background(), timeout)+ defer cancel()++ for {+ select {+ case <-ctx.Done():+ return false+ default:+ }++ req, err := http.NewRequestWithContext(ctx, "GET", base+"/health", nil)+ if err != nil {+ continue+ }+ resp, err := http.DefaultClient.Do(req)+ if err == nil {+ healthy := resp.StatusCode == 200+ _ = resp.Body.Close()+ if healthy {+ return true+ }+ }+ time.Sleep(500 * time.Millisecond)+ }+}++// FindRepoRoot walks up from the current directory to find go.mod.+func FindRepoRoot() string {+ dir, _ := os.Getwd()+ for {+ if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {+ return dir+ }+ parent := filepath.Dir(dir)+ if parent == dir {+ break+ }+ dir = parent+ }+ return filepath.Join("..", "..")+}++// LaunchInstance creates and waits for a test instance to be ready.+func LaunchInstance(base string) (string, error) {+ resp, err := http.Post(+ base+"/instances/launch",+ "application/json",+ strings.NewReader(`{"mode":"headless"}`),+ )+ if err != nil {+ return "", err+ }+ defer resp.Body.Close() //nolint:errcheck++ if resp.StatusCode != 201 && resp.StatusCode != 200 {+ body, _ := io.ReadAll(resp.Body)+ return "", fmt.Errorf("launch failed with status %d: %s", resp.StatusCode, string(body))+ }++ var result map[string]any+ body, _ := io.ReadAll(resp.Body)+ if err := json.Unmarshal(body, &result); err != nil {+ return "", fmt.Errorf("parse launch response: %w", err)+ }++ id, ok := result["id"].(string)+ if !ok {+ return "", fmt.Errorf("no instance id in launch response: %v", result)+ }++ fmt.Fprintf(os.Stderr, "testutil: launched instance %s\n", id)++ // Wait for instance readiness+ ctx, cancel := context.WithTimeout(context.Background(), InstanceTimeout)+ defer cancel()++ for {+ select {+ case <-ctx.Done():+ return "", fmt.Errorf("instance %s did not become ready within %v", id, InstanceTimeout)+ default:+ }++ openResp, err := http.Post(+ base+"/instances/"+id+"/tabs/open",+ "application/json",+ strings.NewReader(`{"url":"about:blank"}`),+ )+ if err != nil {+ time.Sleep(500 * time.Millisecond)+ continue+ }++ var tabID string+ if openResp.StatusCode == 200 {+ openBody, _ := io.ReadAll(openResp.Body)+ var open map[string]any+ if err := json.Unmarshal(openBody, &open); err == nil {+ if v, ok := open["tabId"].(string); ok {+ tabID = v+ }+ }+ }+ _ = openResp.Body.Close()++ if tabID == "" {+ time.Sleep(500 * time.Millisecond)+ continue+ }++ navResp, err := http.Post(+ base+"/tabs/"+tabID+"/navigate",+ "application/json",+ strings.NewReader(`{"url":"about:blank"}`),+ )+ if err != nil {+ time.Sleep(500 * time.Millisecond)+ continue+ }+ status := navResp.StatusCode+ _ = navResp.Body.Close()++ if status == 200 {+ fmt.Fprintf(os.Stderr, "testutil: instance %s is ready\n", id)+ return id, nil+ }+ time.Sleep(500 * time.Millisecond)+ }+}++func filterEnv(env []string, prefixes ...string) []string {+ out := make([]string, 0, len(env))+ for _, e := range env {+ skip := false+ for _, p := range prefixes {+ if strings.HasPrefix(e, p) {+ skip = true+ break+ }+ }+ if !skip {+ out = append(out, e)+ }+ }+ return out+}++func boolStr(b bool) string {+ if b {+ return "true"+ }+ return "false"+}