Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

6.6 KB
Raw Blame
package bench

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

func validateAPIKeys(provider string) error {
	if provider == "fake" {
		return nil
	}
	hasOpenAI := strings.TrimSpace(os.Getenv("OPENAI_API_KEY")) != ""
	hasAnthropic := strings.TrimSpace(os.Getenv("ANTHROPIC_API_KEY")) != ""
	if !hasOpenAI && !hasAnthropic {
		return fmt.Errorf("OPENAI_API_KEY or ANTHROPIC_API_KEY is required")
	}
	return nil
}

func setupLaneContainer(lane Lane, benchDir string, stdout, stderr io.Writer) error {
	if lane == LanePinchtab {
		return setupPinchtabContainer(benchDir, stdout, stderr)
	}
	return setupAgentBrowserContainer(benchDir, stdout, stderr)
}

func setupPinchtabContainer(benchDir string, stdout, stderr io.Writer) error {
	if os.Getenv("BENCHMARK_SKIP_PINCHTAB_RESTART") != "" {
		_, _ = fmt.Fprintln(stdout, "Skipping pinchtab container setup (BENCHMARK_SKIP_PINCHTAB_RESTART=1)")
		return nil
	}

	_, _ = fmt.Fprintln(stdout, "Rebuilding pinchtab with pinchtab-benchmark.json (wrapContent=false)...")

	cmd := exec.Command("docker", "compose",
		"-f", "docker-compose.yml",
		"-f", "docker-compose.benchmark.yml",
		"up", "-d", "--build", "--force-recreate", "--no-deps", "pinchtab")
	cmd.Dir = benchDir
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to rebuild pinchtab: %w", err)
	}

	if err := waitForPinchtabHealth(30 * time.Second); err != nil {
		return err
	}

	if err := verifyPinchtabConfig(); err != nil {
		return err
	}

	_, _ = fmt.Fprintln(stdout, "Verified: pinchtab running with wrapContent=false")
	return nil
}

func setupAgentBrowserContainer(benchDir string, stdout, stderr io.Writer) error {
	if os.Getenv("BENCHMARK_SKIP_AGENT_BROWSER_RESTART") != "" {
		_, _ = fmt.Fprintln(stdout, "Skipping agent-browser container setup (BENCHMARK_SKIP_AGENT_BROWSER_RESTART=1)")
		return nil
	}

	_, _ = fmt.Fprintln(stdout, "Force-recreating agent-browser and fixtures...")

	cmd := exec.Command("docker", "compose",
		"up", "-d", "--build", "--force-recreate", "--no-deps", "fixtures", "agent-browser")
	cmd.Dir = benchDir
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to recreate agent-browser: %w", err)
	}

	if err := waitForFixturesReachable(benchDir, 30*time.Second); err != nil {
		return err
	}

	_, _ = fmt.Fprintln(stdout, "Verified: agent-browser up and fixtures reachable")
	return nil
}

func waitForPinchtabHealth(timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	client := &http.Client{Timeout: 2 * time.Second}

	for time.Now().Before(deadline) {
		req, _ := http.NewRequest("GET", "http://localhost:9867/health", nil)
		req.Header.Set("Authorization", "Bearer benchmark-token")
		resp, err := client.Do(req)
		if err == nil {
			_ = resp.Body.Close()
			if resp.StatusCode == 200 {
				return nil
			}
		}
		time.Sleep(1 * time.Second)
	}
	return fmt.Errorf("pinchtab health check timed out after %v", timeout)
}

func verifyPinchtabConfig() error {
	cmd := exec.Command("docker", "exec", "tools-pinchtab-1",
		"sh", "-c", `echo "${PINCHTAB_CONFIG:-}"`)
	output, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("failed to check pinchtab config: %w", err)
	}

	configPath := strings.TrimSpace(string(output))
	if configPath != "/fixture-config/pinchtab-benchmark.json" {
		return fmt.Errorf("pinchtab not running benchmark config (got %s)", configPath)
	}

	cmd = exec.Command("docker", "exec", "tools-pinchtab-1",
		"cat", "/fixture-config/pinchtab-benchmark.json")
	output, err = cmd.Output()
	if err != nil {
		return fmt.Errorf("failed to read pinchtab config: %w", err)
	}

	if !strings.Contains(string(output), `"wrapContent": false`) &&
		!strings.Contains(string(output), `"wrapContent":false`) {
		return fmt.Errorf("pinchtab config does not have wrapContent=false")
	}

	return nil
}

func waitForFixturesReachable(benchDir string, timeout time.Duration) error {
	deadline := time.Now().Add(timeout)

	for time.Now().Before(deadline) {
		cmd := exec.Command("docker", "compose", "exec", "-T", "agent-browser",
			"curl", "-sf", "http://fixtures/")
		cmd.Dir = benchDir
		if err := cmd.Run(); err == nil {
			return nil
		}
		time.Sleep(1 * time.Second)
	}
	return fmt.Errorf("fixtures not reachable from agent-browser after %v", timeout)
}

func initializeLaneGo(resultsDir string, lane Lane, provider, model string, stdout, stderr io.Writer) (string, error) {
	_ = os.MkdirAll(resultsDir, 0755)

	timestamp := time.Now().Format("20060102_150405")

	var reportFile string
	var reportType string
	if lane == LanePinchtab {
		reportFile = filepath.Join(resultsDir, fmt.Sprintf("pinchtab_benchmark_%s.json", timestamp))
		reportType = "pinchtab"
	} else {
		reportFile = filepath.Join(resultsDir, fmt.Sprintf("agent_browser_benchmark_%s.json", timestamp))
		reportType = "agent-browser"
	}

	runNumber := 1 // Could count existing reports if needed

	report := fmt.Sprintf(`{
  "benchmark": {
    "type": "%s",
    "run_number": %d,
    "timestamp": "%s",
    "model": "%s",
    "runner": "%s"
  },
  "totals": {
    "input_tokens": 0,
    "output_tokens": 0,
    "total_tokens": 0,
    "estimated_cost_usd": 0,
    "tool_calls": 0,
    "steps_passed": 0,
    "steps_failed": 0,
    "steps_skipped": 0,
    "steps_answered": 0,
    "steps_verified_passed": 0,
    "steps_verified_failed": 0,
    "steps_verified_skipped": 0,
    "steps_pending_verification": 0
  },
  "run_usage": {
    "source": "none",
    "provider": "",
    "request_count": 0,
    "input_tokens": 0,
    "output_tokens": 0,
    "cache_creation_input_tokens": 0,
    "cache_read_input_tokens": 0,
    "total_input_tokens": 0,
    "total_tokens": 0
  },
  "steps": []
}`, reportType, runNumber, timestamp, model, runnerSource(provider))

	if err := os.WriteFile(reportFile, []byte(report), 0644); err != nil {
		return "", fmt.Errorf("failed to create report: %w", err)
	}

	ptrFile := filepath.Join(resultsDir, "current_pinchtab_report.txt")
	if lane == LaneAgentBrowser {
		ptrFile = filepath.Join(resultsDir, "current_agent_browser_report.txt")
	}
	if err := os.WriteFile(ptrFile, []byte(reportFile+"\n"), 0644); err != nil {
		return "", fmt.Errorf("failed to write pointer: %w", err)
	}

	// Clear timing state from prior runs
	key := strings.ReplaceAll(reportType, "_", "-")
	_ = os.Remove(filepath.Join(resultsDir, fmt.Sprintf("run_start_%s.ms", key)))
	_ = os.Remove(filepath.Join(resultsDir, fmt.Sprintf("last_step_end_%s.ms", key)))

	_, _ = fmt.Fprintf(stdout, "=== PinchTab Benchmark Run #%d ===\n", runNumber)
	_, _ = fmt.Fprintf(stdout, "Timestamp: %s\n", timestamp)
	_, _ = fmt.Fprintf(stdout, "Report: %s\n\n", reportFile)

	return reportFile, nil
}