Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

11.9 KB
Raw Blame
package orchestrator

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"strings"
	"time"

	"github.com/pinchtab/pinchtab/internal/bridge"
	"github.com/pinchtab/pinchtab/internal/config"
)

// ErrUnknownBrowser disambiguates 400 (unknown target) from 502 (exhaustion) for handlers.
var ErrUnknownBrowser = errors.New("unknown browser target")

// UnknownBrowserError satisfies errors.Is(ErrUnknownBrowser).
type UnknownBrowserError struct {
	Target string
	Err    error
}

func (e *UnknownBrowserError) Error() string {
	return fmt.Sprintf("launch with fallback: resolve target %q: %v", e.Target, e.Err)
}

func (e *UnknownBrowserError) Unwrap() error { return e.Err }

func (e *UnknownBrowserError) Is(target error) bool {
	return target == ErrUnknownBrowser
}

type FallbackAttempt struct {
	Target string              `json:"target"`
	Reason LaunchFailureReason `json:"reason"`
}

// FallbackExhaustedError maps to HTTP 502 browser_target_unavailable.
type FallbackExhaustedError struct {
	Attempts []FallbackAttempt
	Cause    error `json:"-"`
}

func (e *FallbackExhaustedError) Error() string {
	parts := make([]string, 0, len(e.Attempts))
	for _, a := range e.Attempts {
		parts = append(parts, fmt.Sprintf("%s:%s", a.Target, string(a.Reason)))
	}
	msg := fmt.Sprintf("all browser targets failed: [%s]", strings.Join(parts, ", "))
	if e.Cause != nil {
		msg += ": " + e.Cause.Error()
	}
	return msg
}

func (e *FallbackExhaustedError) Unwrap() error { return e.Cause }

// launchWithFallbackTimeout covers the monitor's instanceStartupTimeout plus a margin.
const launchWithFallbackTimeout = 60 * time.Second

var launchOutcomePollInterval = 50 * time.Millisecond

// ResolvedLaunchTarget is the target/provider pair the planner passes into a
// launch attempt.
type ResolvedLaunchTarget struct {
	Name     string
	Provider string
}

type LaunchOutcome struct {
	Status string
	Reason LaunchFailureReason
}

// PlannedLaunch is the planner result before orchestrator-specific instance
// storage metadata has been applied.
type PlannedLaunch struct {
	Instance       *bridge.Instance
	FallbackFrom   string
	FallbackReason LaunchFailureReason
}

// Launcher is the dependency boundary for fallback planning. Implementations
// provide target resolution, launch execution, outcome polling, and teardown.
type Launcher interface {
	ResolveTarget(candidate string) (ResolvedLaunchTarget, error)
	Launch(name, port string, headless bool, opts LaunchOptions) (*bridge.Instance, error)
	WaitForLaunchOutcome(instanceID string, timeout time.Duration) (LaunchOutcome, error)
	TearDownFailedAttempt(instanceID string)
}

// LaunchPlanner contains fallback policy without depending on Orchestrator
// internals. It is intentionally small so policy tests can use a fake Launcher.
type LaunchPlanner struct {
	Launcher Launcher
	Timeout  time.Duration
}

type fallbackAttempt struct {
	target string
	reason LaunchFailureReason
}

func (a fallbackAttempt) String() string {
	return fmt.Sprintf("%s:%s", a.target, string(a.reason))
}

// LaunchWithFallback tries candidates in order, tearing down recoverable failures before moving on.
// Returns *UnknownBrowserError for invalid names, *FallbackExhaustedError when all candidates fail.
func (p LaunchPlanner) LaunchWithFallback(
	name, port string, headless bool,
	primaryAndFallbacks []string,
	opts LaunchOptions,
) (*PlannedLaunch, error) {
	if p.Launcher == nil {
		return nil, fmt.Errorf("launch with fallback: launcher is required")
	}
	timeout := p.Timeout
	if timeout <= 0 {
		timeout = launchWithFallbackTimeout
	}
	candidates := dedupCandidates(primaryAndFallbacks)
	if len(candidates) == 0 {
		return nil, fmt.Errorf("launch with fallback: at least one browser target candidate is required")
	}

	var (
		attempts        []fallbackAttempt
		firstFailTarget string
		firstFailReason LaunchFailureReason
	)
	recordAttempt := func(i int, candidate string, reason LaunchFailureReason) {
		attempts = append(attempts, fallbackAttempt{target: candidate, reason: reason})
		if i == 0 {
			firstFailTarget = candidate
			firstFailReason = reason
		}
	}

	for i, candidate := range candidates {
		resolved, err := p.Launcher.ResolveTarget(candidate)
		if err != nil {
			return nil, &UnknownBrowserError{Target: candidate, Err: err}
		}

		attemptOpts := opts
		attemptOpts.Browser = resolved.Provider
		attemptOpts.TargetName = resolved.Name

		inst, launchErr := p.Launcher.Launch(name, port, headless, attemptOpts)
		if launchErr != nil {
			reason := ClassifyLaunchFailure(launchErr)
			recordAttempt(i, candidate, reason)
			if !IsRecoverable(reason) {
				return nil, fallbackExhaustedError(attempts, fmt.Errorf(
					"launch with fallback: candidate %q failed non-recoverably (%s): %w",
					candidate, string(reason), launchErr,
				))
			}
			continue
		}
		if inst == nil {
			recordAttempt(i, candidate, ReasonUnknown)
			return nil, fallbackExhaustedError(attempts, fmt.Errorf(
				"launch with fallback: candidate %q returned nil instance",
				candidate,
			))
		}

		outcome, waitErr := p.Launcher.WaitForLaunchOutcome(inst.ID, timeout)
		if waitErr != nil {
			reason := ClassifyLaunchFailure(waitErr)
			recordAttempt(i, candidate, reason)
			p.Launcher.TearDownFailedAttempt(inst.ID)
			if !IsRecoverable(reason) {
				return nil, fallbackExhaustedError(attempts, fmt.Errorf(
					"launch with fallback: wait for %q: %w", candidate, waitErr,
				))
			}
			continue
		}

		if outcome.Status == "running" {
			result := &PlannedLaunch{Instance: inst}
			if i > 0 {
				result.FallbackFrom = firstFailTarget
				result.FallbackReason = firstFailReason
			}
			return result, nil
		}

		recordAttempt(i, candidate, outcome.Reason)

		if !IsRecoverable(outcome.Reason) {
			p.Launcher.TearDownFailedAttempt(inst.ID)
			return nil, fallbackExhaustedError(attempts, fmt.Errorf(
				"launch with fallback: candidate %q failed non-recoverably (%s)",
				candidate, string(outcome.Reason),
			))
		}

		p.Launcher.TearDownFailedAttempt(inst.ID)
	}

	return nil, fallbackExhaustedError(attempts, nil)
}

func (o *Orchestrator) LaunchWithFallback(
	name, port string, headless bool,
	primaryAndFallbacks []string,
	opts LaunchOptions,
) (*bridge.Instance, error) {
	planner := LaunchPlanner{
		Launcher: o.fallbackLaunchPlannerLauncher(),
		Timeout:  launchWithFallbackTimeout,
	}
	planned, err := planner.LaunchWithFallback(name, port, headless, primaryAndFallbacks, opts)
	if err != nil {
		return nil, err
	}
	return o.snapshotWithFallbackMetadata(planned), nil
}

func (o *Orchestrator) fallbackLaunchPlannerLauncher() Launcher {
	if o != nil && o.fallbackLauncher != nil {
		return o.fallbackLauncher
	}
	return orchestratorFallbackLauncher{orch: o}
}

type orchestratorFallbackLauncher struct {
	orch *Orchestrator
}

func (l orchestratorFallbackLauncher) ResolveTarget(candidate string) (ResolvedLaunchTarget, error) {
	resolved, err := config.ResolveExplicitBrowserTarget(l.orch.cfg(), candidate)
	if err != nil {
		return ResolvedLaunchTarget{}, err
	}
	return ResolvedLaunchTarget{Name: resolved.Name, Provider: resolved.Provider}, nil
}

func (l orchestratorFallbackLauncher) Launch(name, port string, headless bool, opts LaunchOptions) (*bridge.Instance, error) {
	return l.orch.LaunchWithOptions(name, port, headless, opts)
}

func (l orchestratorFallbackLauncher) WaitForLaunchOutcome(instanceID string, timeout time.Duration) (LaunchOutcome, error) {
	status, reason, err := l.orch.waitForLaunchOutcome(instanceID, timeout)
	return LaunchOutcome{Status: status, Reason: reason}, err
}

func (l orchestratorFallbackLauncher) TearDownFailedAttempt(instanceID string) {
	l.orch.tearDownFailedAttempt(instanceID)
}

func (o *Orchestrator) snapshotWithFallbackMetadata(planned *PlannedLaunch) *bridge.Instance {
	if planned == nil || planned.Instance == nil {
		return nil
	}
	if planned.FallbackFrom == "" {
		return planned.Instance
	}

	reason := string(planned.FallbackReason)
	o.mu.Lock()
	if internal, ok := o.instances[planned.Instance.ID]; ok {
		internal.FallbackFrom = planned.FallbackFrom
		internal.FallbackReason = reason
		snapshot := internal.Instance
		o.mu.Unlock()
		o.syncInstanceToManager(&snapshot)
		return &snapshot
	}
	o.mu.Unlock()

	snapshot := *planned.Instance
	snapshot.FallbackFrom = planned.FallbackFrom
	snapshot.FallbackReason = reason
	return &snapshot
}

func fallbackExhaustedError(attempts []fallbackAttempt, cause error) *FallbackExhaustedError {
	exhausted := &FallbackExhaustedError{
		Attempts: make([]FallbackAttempt, 0, len(attempts)),
		Cause:    cause,
	}
	for _, a := range attempts {
		exhausted.Attempts = append(exhausted.Attempts, FallbackAttempt{Target: a.target, Reason: a.reason})
	}
	return exhausted
}

// waitForLaunchOutcome polls until the instance reaches running/error or the timeout fires.
func (o *Orchestrator) waitForLaunchOutcome(instanceID string, timeout time.Duration) (string, LaunchFailureReason, error) {
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	ticker := time.NewTicker(launchOutcomePollInterval)
	defer ticker.Stop()

	for {
		o.mu.RLock()
		inst, ok := o.instances[instanceID]
		var status string
		var reason LaunchFailureReason
		if ok {
			status = inst.Status
			reason = inst.lastFailureReason
		}
		o.mu.RUnlock()

		if !ok {
			return "", ReasonUnknown, fmt.Errorf("instance %q disappeared from orchestrator map", instanceID)
		}

		switch status {
		case "running":
			return "running", ReasonUnknown, nil
		case "error":
			return "error", reason, nil
		}

		select {
		case <-ctx.Done():
			return "", ReasonUnknown, ctx.Err()
		case <-ticker.C:
		}
	}
}

// failedAttemptExitWait bounds how long fallback waits for a failed attempt's
// process to exit before launching the next candidate. Var so tests can shrink it.
var failedAttemptExitWait = 3 * time.Second

// tearDownFailedAttempt logs but does not propagate errors so teardown never
// blocks fallback indefinitely. It does wait (bounded) for the failed process
// to exit: the next candidate reuses the same profile name/dir, and a
// still-dying browser holding Chrome's SingletonLock would poison the retry.
func (o *Orchestrator) tearDownFailedAttempt(instanceID string) {
	inst := o.detachFailedAttempt(instanceID)
	if inst == nil {
		return
	}
	o.detachedStops.Add(1)
	go func() {
		defer o.detachedStops.Done()
		o.stopDetachedFailedAttempt(instanceID, inst)
	}()
	if inst.cmd != nil {
		if pid := inst.cmd.PID(); pid > 0 && !waitForProcessExit(pid, failedAttemptExitWait) {
			slog.Warn("failed fallback attempt still running; next candidate may hit a locked profile", "id", instanceID, "pid", pid)
		}
	}
}

func (o *Orchestrator) detachFailedAttempt(instanceID string) *InstanceInternal {
	o.mu.Lock()
	inst, ok := o.instances[instanceID]
	if !ok {
		o.mu.Unlock()
		return nil
	}
	inst.Status = "stopping"
	delete(o.instances, instanceID)
	o.mu.Unlock()

	o.removeInstanceFromManager(instanceID)
	o.releaseInstancePorts(instanceID, inst)
	return inst
}

func (o *Orchestrator) stopDetachedFailedAttempt(instanceID string, inst *InstanceInternal) {
	if inst == nil {
		return
	}
	defer func() {
		o.cleanupDetachedFailedAttemptProfile(inst.ProfileName, inst.browser)
	}()

	if inst.cmd == nil {
		if inst.AttachType == "bridge" {
			o.requestChildShutdown(inst)
		}
		return
	}

	if !o.stopChildProcess(instanceID, inst) {
		slog.Warn("failed fallback attempt still running after teardown", "id", instanceID, "pid", inst.cmd.PID())
	}
}

func (o *Orchestrator) cleanupDetachedFailedAttemptProfile(profileName, browser string) {
	o.mu.RLock()
	for _, inst := range o.instances {
		if inst != nil && inst.ProfileName == profileName && instanceIsActive(inst) {
			o.mu.RUnlock()
			return
		}
	}
	o.mu.RUnlock()
	o.cleanupStoppedProfile(profileName, browser)
}

func dedupCandidates(in []string) []string {
	out := make([]string, 0, len(in))
	seen := make(map[string]struct{}, len(in))
	for _, raw := range in {
		name := strings.TrimSpace(raw)
		if name == "" {
			continue
		}
		if _, dup := seen[name]; dup {
			continue
		}
		seen[name] = struct{}{}
		out = append(out, name)
	}
	return out
}