Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

21.4 KB
Raw Blame
package bench

import (
	"encoding/csv"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const defaultBrowserBenchCSV = "https://raw.githubusercontent.com/Halluminate/browserbench/main/browserbench.csv"

var errBrowserBenchHelp = errors.New("browserbench help")

type BrowserBenchArgs struct {
	Provider        Provider
	Model           string
	CSVFile         string
	Output          string
	Tasks           int
	TaskID          string
	SkipInit        bool
	Verbose         bool
	MaxTokens       int
	Temperature     float64
	MaxTurns        int
	TimeoutSeconds  int
	TurnDelayMs     int
	MaxInputTokens  int
	MaxOutputTokens int
}

type BrowserBenchTask struct {
	TaskID          string
	StartingURL     string
	TaskDescription string
	GroundTruthURL  string
	GroundTruth     string
}

type BrowserBenchRow struct {
	TaskID                   string
	StartingURL              string
	TaskDescription          string
	GroundTruthURL           string
	GroundTruth              string
	Provider                 string
	Timestamp                string
	Success                  bool
	ErrorMessage             string
	AgentResult              string
	SessionURL               string
	ExecutionTimeSeconds     string
	PinchTabVersion          string
	StealthLevel             string
	SolverUsed               string
	SolveAttempts            string
	HARPath                  string
	ScreenshotPath           string
	ConsoleLogPath           string
	CommandLogPath           string
	RequestCount             string
	InputTokens              string
	OutputTokens             string
	CacheCreationInputTokens string
	CacheReadInputTokens     string
	TotalInputTokens         string
	TotalTokens              string
}

func defaultBrowserBenchArgs() BrowserBenchArgs {
	return BrowserBenchArgs{
		CSVFile:         defaultBrowserBenchCSV,
		MaxTokens:       defaultMaxTokens,
		Temperature:     defaultTemperature,
		MaxTurns:        80,
		TimeoutSeconds:  defaultTimeoutSeconds,
		TurnDelayMs:     750,
		MaxInputTokens:  0,
		MaxOutputTokens: 0,
	}
}

func parseBrowserBenchArgs(argv []string) (BrowserBenchArgs, error) {
	a := defaultBrowserBenchArgs()
	next := func(i *int, name string) (string, error) {
		*i++
		if *i >= len(argv) {
			return "", fmt.Errorf("%s requires a value", name)
		}
		return argv[*i], nil
	}

	for i := 0; i < len(argv); i++ {
		switch argv[i] {
		case "--help", "-h":
			return a, errBrowserBenchHelp
		case "--provider":
			if err := parseStringFlag(next, &i, argv[i], func(v string) { a.Provider = Provider(v) }); err != nil {
				return a, err
			}
		case "--model":
			if err := parseStringFlag(next, &i, argv[i], func(v string) { a.Model = v }); err != nil {
				return a, err
			}
		case "--csv-file":
			if err := parseStringFlag(next, &i, argv[i], func(v string) { a.CSVFile = v }); err != nil {
				return a, err
			}
		case "--output":
			if err := parseStringFlag(next, &i, argv[i], func(v string) { a.Output = v }); err != nil {
				return a, err
			}
		case "--tasks":
			if err := parseIntFlag(next, &i, argv[i], func(n int) { a.Tasks = n }); err != nil {
				return a, err
			}
		case "--task-id":
			if err := parseStringFlag(next, &i, argv[i], func(v string) { a.TaskID = v }); err != nil {
				return a, err
			}
		case "--skip-init":
			a.SkipInit = true
		case "--verbose", "-v":
			a.Verbose = true
		case "--max-tokens":
			if err := parseIntFlag(next, &i, argv[i], func(n int) { a.MaxTokens = n }); err != nil {
				return a, err
			}
		case "--temperature":
			if err := parseFloatFlag(next, &i, argv[i], func(f float64) { a.Temperature = f }); err != nil {
				return a, err
			}
		case "--max-turns":
			if err := parseIntFlag(next, &i, argv[i], func(n int) { a.MaxTurns = n }); err != nil {
				return a, err
			}
		case "--timeout-seconds":
			if err := parseIntFlag(next, &i, argv[i], func(n int) { a.TimeoutSeconds = n }); err != nil {
				return a, err
			}
		case "--turn-delay-ms":
			if err := parseIntFlag(next, &i, argv[i], func(n int) { a.TurnDelayMs = n }); err != nil {
				return a, err
			}
		case "--max-input-tokens":
			if err := parseIntFlag(next, &i, argv[i], func(n int) { a.MaxInputTokens = n }); err != nil {
				return a, err
			}
		case "--max-output-tokens":
			if err := parseIntFlag(next, &i, argv[i], func(n int) { a.MaxOutputTokens = n }); err != nil {
				return a, err
			}
		default:
			return a, fmt.Errorf("unknown flag %q", argv[i])
		}
	}

	if a.Tasks < 0 {
		return a, fmt.Errorf("--tasks must be >= 0")
	}
	return a, nil
}

func writeBrowserBenchUsage(w io.Writer) {
	_, _ = fmt.Fprint(w, `Usage:
  runner browserbench [options]

Options:
`+usageLineProvider+
		usageLineModel+
		`  --csv-file PATH_OR_URL        BrowserBench CSV (default: Halluminate raw CSV)
  --output PATH                 Output CSV path (default: timestamped in tests/tools/browserbench/results)
  --tasks N                     Limit number of tasks
  --task-id ID                  Run only one task_id
  --skip-init                   Reuse existing benchmark container
`+usageLineMaxTokens+
		usageLineTemperature+
		usageLineMaxTurns+
		usageLineTimeoutSeconds+
		usageLineTurnDelayMs+
		`  --max-input-tokens N
  --max-output-tokens N
  --verbose, -v

Example:
  go run ./tests/tools/runner browserbench --provider anthropic --tasks 5 --verbose
`)
}

func RunBrowserBench(argv []string, stdout, stderr io.Writer) int {
	args, err := parseBrowserBenchArgs(argv)
	if err != nil {
		if errors.Is(err, errBrowserBenchHelp) {
			writeBrowserBenchUsage(stdout)
			return 0
		}
		_, _ = fmt.Fprintf(stderr, "runner browserbench: %v\n\n", err)
		writeBrowserBenchUsage(stderr)
		return 1
	}

	provider := resolveProvider(Args{Provider: args.Provider})
	if err := validateAPIKeys(provider); err != nil {
		_, _ = fmt.Fprintf(stderr, "runner browserbench: %v\n", err)
		return 1
	}
	model := resolveModel(provider, args.Model)
	toolsDir := resolveToolsDir()

	if !args.SkipInit {
		if err := setupPinchtabContainer(toolsDir, stdout, stderr); err != nil {
			_, _ = fmt.Fprintf(stderr, "runner browserbench: %v\n", err)
			return 1
		}
	}

	tasks, err := loadBrowserBenchTasks(args.CSVFile)
	if err != nil {
		_, _ = fmt.Fprintf(stderr, "runner browserbench: load CSV: %v\n", err)
		return 1
	}
	tasks = selectBrowserBenchTasks(tasks, args.TaskID, args.Tasks)
	if len(tasks) == 0 {
		_, _ = fmt.Fprintln(stderr, "runner browserbench: no tasks selected")
		return 1
	}

	resultsDir := filepath.Join(toolsDir, "browserbench", "results")
	if err := os.MkdirAll(resultsDir, 0o755); err != nil {
		_, _ = fmt.Fprintf(stderr, "runner browserbench: create results dir: %v\n", err)
		return 1
	}

	outputPath := args.Output
	if outputPath == "" {
		outputPath = filepath.Join(resultsDir, fmt.Sprintf("pinchtab_browserbench_%s.csv", time.Now().Format("20060102_150405")))
	}
	runDir := strings.TrimSuffix(outputPath, filepath.Ext(outputPath))
	if err := os.MkdirAll(runDir, 0o755); err != nil {
		_, _ = fmt.Fprintf(stderr, "runner browserbench: create run dir: %v\n", err)
		return 1
	}

	shell, err := NewPersistentShell(toolsDir)
	if err != nil {
		_, _ = fmt.Fprintf(stderr, "runner browserbench: shell init failed: %v\n", err)
		return 1
	}
	defer shell.Close(false)

	pinchtabVersion := detectPinchTabVersion(shell)
	rows := make([]BrowserBenchRow, 0, len(tasks))

	for i, task := range tasks {
		_, _ = fmt.Fprintf(stdout, "[browserbench] task %d/%d id=%s url=%s\n", i+1, len(tasks), task.TaskID, task.StartingURL)
		row, err := runBrowserBenchTask(args, provider, model, task, runDir, shell, pinchtabVersion, stdout, stderr)
		if err != nil {
			row.ErrorMessage = err.Error()
		}
		rows = append(rows, row)
		if err := writeBrowserBenchCSV(outputPath, rows); err != nil {
			_, _ = fmt.Fprintf(stderr, "runner browserbench: write CSV: %v\n", err)
			return 1
		}
	}

	passed := 0
	for _, row := range rows {
		if row.Success {
			passed++
		}
	}
	_, _ = fmt.Fprintf(stdout, "[browserbench] wrote %s (%d/%d passed)\n", outputPath, passed, len(rows))
	return 0
}

func loadBrowserBenchTasks(path string) ([]BrowserBenchTask, error) {
	data, err := readFileOrURL(path)
	if err != nil {
		return nil, err
	}
	r := csv.NewReader(strings.NewReader(string(data)))
	records, err := r.ReadAll()
	if err != nil {
		return nil, err
	}
	if len(records) < 2 {
		return nil, fmt.Errorf("CSV has no data rows")
	}
	headers := make(map[string]int)
	for i, h := range records[0] {
		headers[strings.TrimSpace(strings.ToLower(h))] = i
	}
	required := []string{"task_id", "starting_url", "task_description", "ground_truth_url", "ground_truth"}
	for _, key := range required {
		if _, ok := headers[key]; !ok {
			return nil, fmt.Errorf("missing required column %q", key)
		}
	}
	var tasks []BrowserBenchTask
	for _, rec := range records[1:] {
		if len(rec) == 0 {
			continue
		}
		tasks = append(tasks, BrowserBenchTask{
			TaskID:          rec[headers["task_id"]],
			StartingURL:     rec[headers["starting_url"]],
			TaskDescription: rec[headers["task_description"]],
			GroundTruthURL:  rec[headers["ground_truth_url"]],
			GroundTruth:     rec[headers["ground_truth"]],
		})
	}
	return tasks, nil
}

func selectBrowserBenchTasks(tasks []BrowserBenchTask, taskID string, limit int) []BrowserBenchTask {
	selected := make([]BrowserBenchTask, 0, len(tasks))
	for _, task := range tasks {
		if taskID != "" && task.TaskID != taskID {
			continue
		}
		selected = append(selected, task)
		if limit > 0 && len(selected) >= limit {
			break
		}
	}
	return selected
}

func runBrowserBenchTask(args BrowserBenchArgs, provider, model string, task BrowserBenchTask, runDir string, shell *PersistentShell, pinchtabVersion string, stdout, stderr io.Writer) (BrowserBenchRow, error) {
	row := BrowserBenchRow{
		TaskID:          task.TaskID,
		StartingURL:     task.StartingURL,
		TaskDescription: task.TaskDescription,
		GroundTruthURL:  task.GroundTruthURL,
		GroundTruth:     task.GroundTruth,
		Provider:        provider,
		Timestamp:       time.Now().UTC().Format(time.RFC3339),
		PinchTabVersion: pinchtabVersion,
	}

	taskDir := filepath.Join(runDir, fmt.Sprintf("task-%s", sanitizeTaskID(task.TaskID)))
	artifactsDir := filepath.Join(taskDir, "artifacts")
	_ = os.MkdirAll(artifactsDir, 0o755)
	commandLogPath := filepath.Join(taskDir, "commands.ndjson")
	row.CommandLogPath = commandLogPath

	if _, _, err := shell.Run(fmt.Sprintf("mkdir -p %q && ./scripts/pt console --clear >/dev/null 2>&1 || true", artifactsDir), 30*time.Second); err != nil {
		return row, err
	}
	_, _, _ = shell.Run(`curl -sS -X POST -H 'Authorization: Bearer benchmark-token' http://localhost:9867/network/clear >/dev/null 2>&1 || true`, 15*time.Second)
	if _, _, err := shell.Run(fmt.Sprintf("./scripts/pt nav %q >/dev/null", task.StartingURL), 45*time.Second); err != nil {
		return row, fmt.Errorf("pre-nav failed: %w", err)
	}

	runner := createRunner(provider, model, Args{
		Provider:        Provider(provider),
		Model:           model,
		MaxTokens:       args.MaxTokens,
		Temperature:     args.Temperature,
		MaxInputTokens:  args.MaxInputTokens,
		MaxOutputTokens: args.MaxOutputTokens,
	})

	before := runner.Usage()
	start := time.Now()
	finalText, loopErr := runBrowserBenchAgentLoop(task, provider, shell, runner, args, commandLogPath, stdout)
	elapsed := time.Since(start)
	after := runner.Usage()
	usage := diffUsage(after, before)

	row.AgentResult = extractFinalAnswer(finalText)
	row.ExecutionTimeSeconds = fmt.Sprintf("%.3f", elapsed.Seconds())
	row.RequestCount = strconv.Itoa(usage.RequestCount)
	row.InputTokens = strconv.Itoa(usage.InputTokens)
	row.OutputTokens = strconv.Itoa(usage.OutputTokens)
	row.CacheCreationInputTokens = strconv.Itoa(usage.CacheCreationInputTokens)
	row.CacheReadInputTokens = strconv.Itoa(usage.CacheReadInputTokens)
	row.TotalInputTokens = strconv.Itoa(usage.InputTokens + usage.CacheCreationInputTokens + usage.CacheReadInputTokens)
	row.TotalTokens = strconv.Itoa(usage.InputTokens + usage.CacheCreationInputTokens + usage.CacheReadInputTokens + usage.OutputTokens)
	if loopErr != nil {
		row.ErrorMessage = loopErr.Error()
	}

	stealthPath := filepath.Join(artifactsDir, "stealth-status.json")
	autosolverPath := filepath.Join(artifactsDir, "autosolver.json")
	screenshotPath := filepath.Join(artifactsDir, "final.png")
	harPath := filepath.Join(artifactsDir, "network.har")
	consolePath := filepath.Join(artifactsDir, "console.log")
	row.ScreenshotPath = screenshotPath
	row.HARPath = harPath
	row.ConsoleLogPath = consolePath

	_, _, _ = shell.Run(fmt.Sprintf("curl -sS -H 'Authorization: Bearer benchmark-token' http://localhost:9867/stealth/status > %q || true", stealthPath), 15*time.Second)
	_, _, _ = shell.Run(fmt.Sprintf("curl -sS -H 'Authorization: Bearer benchmark-token' http://localhost:9867/config/autosolver > %q || true", autosolverPath), 15*time.Second)
	_, _, _ = shell.Run(fmt.Sprintf("curl -sS -H 'Authorization: Bearer benchmark-token' 'http://localhost:9867/screenshot?format=png&raw=true' > %q || true", screenshotPath), 30*time.Second)
	_, _, _ = shell.Run(fmt.Sprintf("curl -sS -H 'Authorization: Bearer benchmark-token' 'http://localhost:9867/network/export?format=har' > %q || true", harPath), 30*time.Second)
	_, _, _ = shell.Run(fmt.Sprintf("./scripts/pt console > %q 2>/dev/null || true", consolePath), 15*time.Second)

	row.StealthLevel = readJSONField(stealthPath, "level")
	row.SolverUsed, row.SolveAttempts = deriveSolveMetadata(commandLogPath)
	row.Success = evaluateBrowserBenchAnswer(row.AgentResult, task.GroundTruth) && row.ErrorMessage == ""

	return row, nil
}

func runBrowserBenchAgentLoop(task BrowserBenchTask, provider string, shell *PersistentShell, runner Runner, args BrowserBenchArgs, commandLogPath string, stdout io.Writer) (string, error) {
	conversation := runner.InitialConversation(browserBenchUserPrompt(task))
	systemPrompt := browserBenchSystemPrompt(filepath.Dir(filepath.Dir(resolveToolsDir())))
	out := NewOutputWriter(stdout, args.Verbose)

	outcome := runTurnLoop(turnLoopConfig{
		maxTurns:        args.MaxTurns,
		turnDelayMs:     args.TurnDelayMs,
		timeoutSeconds:  args.TimeoutSeconds,
		maxInputTokens:  args.MaxInputTokens,
		maxOutputTokens: args.MaxOutputTokens,
		spinnerMessage:  "Solving BrowserBench task...",
		systemPrompt:    systemPrompt,
		commandLog:      commandLogPath,
		provider:        provider,
		compactionSummary: func() string {
			return "BrowserBench task in progress. Continue from the current browser state until you can answer with FINAL_ANSWER."
		},
		onToolResults: nil,
	}, runner, shell, out, conversation)

	switch outcome.stop {
	case stopFinal:
		return outcome.finalText, nil
	case stopBudgetInput:
		return "", fmt.Errorf("input token budget exceeded")
	case stopBudgetOutput:
		return "", fmt.Errorf("output token budget exceeded")
	case stopAPIError:
		return "", outcome.err
	default: // stopMaxTurns
		return "", fmt.Errorf("max turns reached (%d)", args.MaxTurns)
	}
}

func browserBenchSystemPrompt(repoRoot string) string {
	base := `You are a precise web-task execution agent running BrowserBench-style tasks through PinchTab.

Rules:
- Use shell tool calls only.
- Use only ./scripts/pt for browser actions.
- The browser is already pointed at the task's starting URL when the task begins.
- Do NOT create or manage PinchTab sessions yourself. Do not call ./scripts/pt session create, session login, or other session bootstrap commands.
- Prefer PinchTab primitives such as snap, text, click, fill, press, wait, back, eval, console, and solve.
- Do NOT invent convenience flags. "--snap" and "--snap-diff" are valid on some actions (for example nav, click, fill, select, scroll, back, forward, reload) but not on every command. In particular, do not use them with press. When unsure, run a separate "./scripts/pt snap -i -c" after the action.
- If a site presents a challenge, you may use ./scripts/pt solve.
- Stay on the starting website domain unless the task explicitly requires a page on the same site reached by navigation.
- Do not fabricate answers.
- When you are confident, end with a single line in this exact format: FINAL_ANSWER: <answer>
- Keep the final answer short and factual.`

	skill := LoadSkillContent(filepath.Join(repoRoot, "skills", "pinchtab", "SKILL.md"))
	if skill == "" {
		return base
	}
	return base + "\n\n# PinchTab Skill\n\n" + skill
}

func browserBenchUserPrompt(task BrowserBenchTask) string {
	return fmt.Sprintf(`Complete this BrowserBench task.

Starting URL: %s
Task ID: %s
Instruction: %s

Important:
- the browser is already opened on the starting URL
- do not use the ground truth in your reasoning
- return exactly one final line: FINAL_ANSWER: <answer>`, task.StartingURL, task.TaskID, task.TaskDescription)
}

func readFileOrURL(path string) ([]byte, error) {
	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
		client := &http.Client{Timeout: 30 * time.Second}
		resp, err := client.Get(path)
		if err != nil {
			return nil, err
		}
		defer func() { _ = resp.Body.Close() }()
		if resp.StatusCode < 200 || resp.StatusCode > 299 {
			return nil, fmt.Errorf("GET %s: status %d", path, resp.StatusCode)
		}
		return io.ReadAll(resp.Body)
	}
	return os.ReadFile(path)
}

func writeBrowserBenchCSV(path string, rows []BrowserBenchRow) error {
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return err
	}
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer func() { _ = f.Close() }()
	w := csv.NewWriter(f)
	headers := []string{
		"task_id", "starting_url", "task_description", "ground_truth_url", "ground_truth",
		"provider", "timestamp", "success", "error_message", "agent_result", "session_url", "execution_time",
		"pinchtab_version", "stealth_level", "solver_used", "attempts",
		"har_path", "screenshot_path", "console_log_path", "command_log_path",
		"request_count", "input_tokens", "output_tokens", "cache_creation_input_tokens", "cache_read_input_tokens", "total_input_tokens", "total_tokens",
	}
	if err := w.Write(headers); err != nil {
		return err
	}
	for _, row := range rows {
		rec := []string{
			row.TaskID, row.StartingURL, row.TaskDescription, row.GroundTruthURL, row.GroundTruth,
			row.Provider, row.Timestamp, strconv.FormatBool(row.Success), row.ErrorMessage, row.AgentResult, row.SessionURL, row.ExecutionTimeSeconds,
			row.PinchTabVersion, row.StealthLevel, row.SolverUsed, row.SolveAttempts,
			row.HARPath, row.ScreenshotPath, row.ConsoleLogPath, row.CommandLogPath,
			row.RequestCount, row.InputTokens, row.OutputTokens, row.CacheCreationInputTokens, row.CacheReadInputTokens, row.TotalInputTokens, row.TotalTokens,
		}
		if err := w.Write(rec); err != nil {
			return err
		}
	}
	w.Flush()
	return w.Error()
}

func detectPinchTabVersion(shell *PersistentShell) string {
	out, _, err := shell.Run("docker exec tools-pinchtab-1 sh -lc 'pinchtab --version 2>/dev/null || true'", 15*time.Second)
	if err != nil {
		return ""
	}
	lines := strings.Split(strings.TrimSpace(out), "\n")
	if len(lines) == 0 {
		return strings.TrimSpace(out)
	}
	return strings.TrimSpace(lines[len(lines)-1])
}

func sanitizeTaskID(v string) string {
	v = strings.TrimSpace(v)
	if v == "" {
		return "unknown"
	}
	re := regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
	return re.ReplaceAllString(v, "-")
}

func extractFinalAnswer(text string) string {
	for _, line := range strings.Split(text, "\n") {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(strings.ToUpper(line), "FINAL_ANSWER:") {
			return strings.TrimSpace(line[len("FINAL_ANSWER:"):])
		}
	}
	return strings.TrimSpace(text)
}

func evaluateBrowserBenchAnswer(answer, truth string) bool {
	a := normalizeAnswer(answer)
	g := normalizeAnswer(truth)
	if a == "" || g == "" {
		return false
	}
	if g == "yes" || g == "no" {
		return a == g || strings.HasPrefix(a, g+" ") || strings.Contains(a, " "+g+" ")
	}
	if a == g || strings.Contains(a, g) || strings.Contains(g, a) {
		return true
	}
	return false
}

func normalizeAnswer(s string) string {
	s = strings.ToLower(strings.TrimSpace(s))
	s = strings.ReplaceAll(s, "’", "'")
	s = strings.ReplaceAll(s, "“", `"`)
	s = strings.ReplaceAll(s, "”", `"`)
	s = regexp.MustCompile(`\s+`).ReplaceAllString(s, " ")
	s = strings.Trim(s, " .,!?:;\"'")
	return s
}

func diffUsage(after, before UsageCounters) UsageCounters {
	return UsageCounters{
		RequestCount:             after.RequestCount - before.RequestCount,
		InputTokens:              after.InputTokens - before.InputTokens,
		OutputTokens:             after.OutputTokens - before.OutputTokens,
		CacheCreationInputTokens: after.CacheCreationInputTokens - before.CacheCreationInputTokens,
		CacheReadInputTokens:     after.CacheReadInputTokens - before.CacheReadInputTokens,
	}
}

func readJSONField(path, key string) string {
	data, err := os.ReadFile(path)
	if err != nil {
		return ""
	}
	var m map[string]any
	if err := json.Unmarshal(data, &m); err != nil {
		return ""
	}
	if v, ok := m[key]; ok {
		return strings.TrimSpace(fmt.Sprint(v))
	}
	return ""
}

func deriveSolveMetadata(commandLogPath string) (string, string) {
	data, err := os.ReadFile(commandLogPath)
	if err != nil {
		return "", ""
	}
	lines := strings.Split(string(data), "\n")
	solver := ""
	maxAttempt := 0
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		var entry struct {
			Output string `json:"output"`
		}
		if err := json.Unmarshal([]byte(line), &entry); err != nil {
			continue
		}
		var payload struct {
			Solver   string `json:"solver"`
			Attempts int    `json:"attempts"`
		}
		if err := json.Unmarshal([]byte(entry.Output), &payload); err != nil {
			continue
		}
		if payload.Solver != "" {
			solver = payload.Solver
		}
		if payload.Attempts > maxAttempt {
			maxAttempt = payload.Attempts
		}
	}
	if maxAttempt == 0 {
		return solver, ""
	}
	return solver, strconv.Itoa(maxAttempt)
}