Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

7.3 KB
Raw Blame
// Command runner is the Go port of
// tests/benchmark/scripts/run-api-benchmark.ts.
//
// Exit codes:
//
//	0  success (or --help)
//	1  argument/setup error
//	2  max turns reached
//	3  idle turn limit reached
//	4  budget exceeded
package bench

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
)

func Run(argv []string, stdout, stderr io.Writer) int {
	if len(argv) > 0 {
		switch argv[0] {
		case "verify-step":
			return RunVerifyStep(argv[1:], stdout, stderr)
		case "record-step":
			return RunRecordStep(argv[1:], stdout, stderr)
		case "step-end":
			return RunStepEnd(argv[1:], stdout, stderr)
		}
	}

	args, err := ParseArgs(argv)
	if err != nil {
		if errors.Is(err, errHelp) {
			WriteUsage(stdout)
			return 0
		}
		_, _ = fmt.Fprintf(stderr, "runner: %v\n\n", err)
		WriteUsage(stderr)
		return 1
	}

	if args.DryRun {
		_, _ = fmt.Fprint(stdout, formatPlan(args))
		return 0
	}

	toolsDir := resolveToolsDir()
	benchmarkDir := resolveBenchmarkDir()
	provider := resolveProvider(args)
	model := resolveModel(provider, args.Model)
	groups := resolveGroupsFromArgs(args)

	// Validate API keys
	if err := validateAPIKeys(provider); err != nil {
		_, _ = fmt.Fprintf(stderr, "runner: %v\n", err)
		return 1
	}

	// Setup container (unless --skip-init)
	if !args.SkipInit {
		if err := setupLaneContainer(args.Lane, toolsDir, stdout, stderr); err != nil {
			_, _ = fmt.Fprintf(stderr, "runner: %v\n", err)
			return 1
		}
	}

	// Initialize report (results go in benchmarkDir)
	resultsDir := filepath.Join(benchmarkDir, "results")
	reportFile := args.ReportFile
	if reportFile == "" {
		if args.SkipInit {
			reportFile = resolveReportPath(resultsDir, args.Lane)
		} else {
			var err error
			reportFile, err = initializeLaneGo(resultsDir, args.Lane, provider, model, stdout, stderr)
			if err != nil {
				_, _ = fmt.Fprintf(stderr, "runner: %v\n", err)
				return 1
			}
		}
	}

	_, _ = fmt.Fprintf(stdout, "[benchmark-runner] provider=%s model=%s lane=%s groups=%s report=%s\n",
		provider, model, args.Lane, formatGroups(groups), reportFile)

	// Harness-generated start-of-run summary
	PrintStartBanner(stdout, args.Lane, benchmarkDir, groups, model, reportFile)

	runner := createRunner(provider, model, args)
	shell, err := NewPersistentShell(toolsDir)
	if err != nil {
		_, _ = fmt.Fprintf(stderr, "runner: shell init failed: %v\n", err)
		return 1
	}
	defer shell.Close(false)

	cfg := LoopConfig{
		Lane:            args.Lane,
		Provider:        provider,
		Model:           model,
		Groups:          groups,
		ReportFile:      reportFile,
		MaxTurns:        args.MaxTurns,
		MaxIdleTurns:    args.MaxIdleTurns,
		TimeoutSeconds:  args.TimeoutSeconds,
		TurnDelayMs:     args.TurnDelayMs,
		Finalize:        args.Finalize,
		ToolsDir:        toolsDir,
		BenchmarkDir:    benchmarkDir,
		CommandLogFile:  filepath.Join(resultsDir, commandLogName(args.Lane)),
		MaxInputTokens:  args.MaxInputTokens,
		MaxOutputTokens: args.MaxOutputTokens,
		TerseSummary:    args.TerseSummary,
		Verbose:         args.Verbose,
		Stdout:          stdout,
		Stderr:          stderr,
	}

	result := RunLoop(cfg, runner, shell)
	return result.ExitCode
}

func resolveRepoRoot() string {
	cwd, _ := os.Getwd()
	dir := cwd
	for {
		if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
			return dir
		}
		parent := filepath.Dir(dir)
		if parent == dir {
			return cwd
		}
		dir = parent
	}
}

func resolveToolsDir() string {
	return filepath.Join(resolveRepoRoot(), "tests", "tools")
}

func resolveBenchmarkDir() string {
	return filepath.Join(resolveRepoRoot(), "tests", "benchmark")
}

func resolveProvider(args Args) string {
	if args.Provider != ProviderUnset {
		return string(args.Provider)
	}
	hasOpenAI := strings.TrimSpace(os.Getenv("OPENAI_API_KEY")) != ""
	hasAnthropic := strings.TrimSpace(os.Getenv("ANTHROPIC_API_KEY")) != ""
	if hasOpenAI && hasAnthropic {
		_, _ = fmt.Fprintln(os.Stderr, "warning: multiple providers configured, defaulting to anthropic")
		return "anthropic"
	}
	if hasOpenAI {
		return "openai"
	}
	if hasAnthropic {
		return "anthropic"
	}
	return "anthropic"
}

func resolveModel(provider, explicit string) string {
	if explicit != "" {
		return explicit
	}
	if provider == "openai" {
		if m := os.Getenv("OPENAI_MODEL"); m != "" {
			return m
		}
		return "gpt-5"
	}
	if m := os.Getenv("ANTHROPIC_MODEL"); m != "" {
		return m
	}
	return "claude-haiku-4-5-20251001"
}

func resolveGroupsFromArgs(args Args) []int {
	if len(args.Groups) > 0 {
		return args.Groups
	}
	switch args.Profile {
	case "common10":
		return []int{0, 1, 2, 3}
	default:
		return nil
	}
}

func commandLogName(lane Lane) string {
	if lane == LaneAgentBrowser {
		return "agent_browser_commands.ndjson"
	}
	return "pinchtab_commands.ndjson"
}

func resolveReportPath(benchDir string, lane Lane) string {
	ptrFile := filepath.Join(benchDir, "results", "current_pinchtab_report.txt")
	if lane == LaneAgentBrowser {
		ptrFile = filepath.Join(benchDir, "results", "current_agent_browser_report.txt")
	}
	data, err := os.ReadFile(ptrFile)
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(data))
}

func runnerSource(provider string) string {
	if provider == "openai" {
		return "openai-responses"
	}
	return "anthropic-messages"
}

func createRunner(provider, model string, args Args) Runner {
	promptCaching := !args.NoPromptCaching
	if provider == "openai" {
		apiKey := os.Getenv("OPENAI_API_KEY")
		return NewOpenAIRunner(apiKey, model, args.MaxTokens, args.Temperature, promptCaching)
	}
	if provider == "fake" {
		return NewFakeRunner(model, nil)
	}
	apiKey := os.Getenv("ANTHROPIC_API_KEY")
	return NewAnthropicRunner(apiKey, model, args.MaxTokens, args.Temperature, promptCaching)
}

func formatPlan(a Args) string {
	var b strings.Builder
	b.WriteString("runner (Go) — resolved plan\n")
	_, _ = fmt.Fprintf(&b, "  lane:              %s\n", a.Lane)
	_, _ = fmt.Fprintf(&b, "  provider:          %s\n", stringOr(string(a.Provider), "(auto-detect from env)"))
	_, _ = fmt.Fprintf(&b, "  model:             %s\n", stringOr(a.Model, "(provider default)"))
	_, _ = fmt.Fprintf(&b, "  groups:            %s\n", formatGroups(a.Groups))
	_, _ = fmt.Fprintf(&b, "  profile:           %s\n", stringOr(a.Profile, "(none)"))
	_, _ = fmt.Fprintf(&b, "  max-tokens:        %d\n", a.MaxTokens)
	_, _ = fmt.Fprintf(&b, "  temperature:       %g\n", a.Temperature)
	_, _ = fmt.Fprintf(&b, "  max-turns:         %d\n", a.MaxTurns)
	_, _ = fmt.Fprintf(&b, "  max-idle-turns:    %d\n", a.MaxIdleTurns)
	_, _ = fmt.Fprintf(&b, "  timeout-seconds:   %d\n", a.TimeoutSeconds)
	_, _ = fmt.Fprintf(&b, "  turn-delay-ms:     %d\n", a.TurnDelayMs)
	_, _ = fmt.Fprintf(&b, "  report-file:       %s\n", stringOr(a.ReportFile, "(auto-generated)"))
	_, _ = fmt.Fprintf(&b, "  skip-init:         %t\n", a.SkipInit)
	_, _ = fmt.Fprintf(&b, "  no-prompt-caching: %t\n", a.NoPromptCaching)
	_, _ = fmt.Fprintf(&b, "  finalize:          %t\n", a.Finalize)
	_, _ = fmt.Fprintf(&b, "  dry-run:           %t\n", a.DryRun)
	_, _ = fmt.Fprintf(&b, "  index-file:        %s\n", stringOr(a.IndexFile, "(default)"))
	return b.String()
}

func stringOr(v, fallback string) string {
	if v == "" {
		return fallback
	}
	return v
}

func formatGroups(groups []int) string {
	if len(groups) == 0 {
		return "all"
	}
	parts := make([]string, len(groups))
	for i, g := range groups {
		parts[i] = fmt.Sprintf("%d", g)
	}
	return strings.Join(parts, ",")
}