High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
refactor: collapse capability switches and split autorestart strategy
Luigi Agostidecfd21fe450
diff --git a/internal/orchestrator/handlers.go b/internal/orchestrator/handlers.goindex 7e623865..496915f7 100644--- a/internal/orchestrator/handlers.go+++ b/internal/orchestrator/handlers.go@@ -7,6 +7,28 @@ import ("github.com/pinchtab/pinchtab/internal/routes")+// Allows reports whether the given capability is permitted by the orchestrator's+// current security settings. Centralises the per-capability dispatch so callers+// don't need their own switch over routes.Capability.+func (o *Orchestrator) Allows(cap routes.Capability) bool {+ switch cap {+ case routes.CapEvaluate:+ return o.AllowsEvaluate()+ case routes.CapMacro:+ return o.AllowsMacro()+ case routes.CapScreencast:+ return o.AllowsScreencast()+ case routes.CapDownload:+ return o.AllowsDownload()+ case routes.CapUpload:+ return o.AllowsUpload()+ case routes.CapStateExport:+ return o.AllowsStateExport()+ default:+ return false+ }+}+func registerCapabilityRoute(mux *http.ServeMux, route string, enabled bool, feature, setting, code string, next http.HandlerFunc) {if enabled {mux.HandleFunc(route, next)diff --git a/internal/strategy/autorestart/autorestart.go b/internal/strategy/autorestart/autorestart.goindex 1721df94..4332bc3a 100644--- a/internal/strategy/autorestart/autorestart.go+++ b/internal/strategy/autorestart/autorestart.go@@ -10,16 +10,10 @@ package autorestartimport ("context"- "fmt"- "log/slog"- "net/http""sync""time"- "github.com/pinchtab/pinchtab/internal/activity"- "github.com/pinchtab/pinchtab/internal/bridge""github.com/pinchtab/pinchtab/internal/config"- "github.com/pinchtab/pinchtab/internal/httpx""github.com/pinchtab/pinchtab/internal/orchestrator""github.com/pinchtab/pinchtab/internal/strategy")@@ -153,15 +147,11 @@ func (s *Strategy) Start(ctx context.Context) error {s.ctx, s.cancel = context.WithCancel(ctx)s.mu.Unlock()- // Subscribe to orchestrator events for crash detection.s.orch.OnEvent(func(evt orchestrator.InstanceEvent) {s.handleEvent(evt)})- // Launch the initial instance.go s.launchInitial()-- // Start the stability checker.go s.stabilityLoop()return nil@@ -178,14 +168,6 @@ func (s *Strategy) Stop() error {return nil}-// RegisterRoutes adds shorthand endpoints that proxy to the managed instance.-func (s *Strategy) RegisterRoutes(mux *http.ServeMux) {- s.orch.RegisterHandlers(mux)- strategy.RegisterShorthandRoutes(mux, s.orch, s.proxyToManaged)- mux.HandleFunc("GET /tabs", s.handleTabs)- mux.HandleFunc("GET "+s.config.StatusPath, s.handleStatus)-}-// State returns the current restart state for observability.func (s *Strategy) State() RestartState {s.mu.Lock()@@ -210,254 +192,6 @@ func (s *Strategy) State() RestartState {}}-// --- Internal -----// launchInitial launches the first instance (or after strategy start).-func (s *Strategy) launchInitial() {- s.mu.Lock()- ctx := s.ctx- s.mu.Unlock()-- if ctx == nil || ctx.Err() != nil {- return- }-- inst, err := s.orch.Launch(s.config.ProfileName, "", s.config.Headless, nil)- if err != nil {- slog.Error(s.logPrefix("initial launch failed"), "profile", s.config.ProfileName, "err", err)- return- }-- s.mu.Lock()- s.instanceID = inst.ID- s.headless = inst.Headless- s.lastStart = time.Now()- s.mu.Unlock()-- slog.Info(s.logPrefix("instance launched"), "id", inst.ID, "profile", s.config.ProfileName)-}--// handleEvent processes orchestrator lifecycle events.-func (s *Strategy) handleEvent(evt orchestrator.InstanceEvent) {- s.mu.Lock()- managedID := s.instanceID- deliberate := s.deliberate- restarting := s.restarting- ctx := s.ctx- s.mu.Unlock()-- // Only handle events for the managed instance.- if evt.Instance == nil || evt.Instance.ID != managedID {- return- }-- // Skip if a restart is already in progress (prevents duplicate handling- // when both instance.error and instance.stopped fire for the same crash).- if restarting {- return- }-- switch evt.Type {- case "instance.stopped":- if deliberate {- slog.Info(s.logPrefix("instance stopped deliberately"), "id", managedID)- return- }- // Instance exited unexpectedly — check if we should restart.- s.handleCrash(ctx, managedID)-- case "instance.error":- if deliberate {- return- }- s.handleCrash(ctx, managedID)- }-}--// handleCrash decides whether to restart the crashed instance.-func (s *Strategy) handleCrash(ctx context.Context, crashedID string) {- if ctx == nil || ctx.Err() != nil {- return- }-- s.mu.Lock()- s.restarting = true- s.restartCount++- s.lastCrash = time.Now()- count := s.restartCount- maxRestarts := s.config.MaxRestarts- backoff := s.config.InitBackoff * time.Duration(1<<uint(count-1))- if backoff > s.config.MaxBackoff {- backoff = s.config.MaxBackoff- }- s.mu.Unlock()-- if s.hasRestartLimit() && count > maxRestarts {- slog.Error(s.logPrefix("max restarts exceeded, giving up"),- "id", crashedID,- "restartCount", count-1,- "maxRestarts", maxRestarts,- )- s.mu.Lock()- s.restarting = false- s.mu.Unlock()- if s.orch != nil {- s.orch.EmitEvent("instance.crashed", &bridge.Instance{- ID: crashedID,- Status: "crashed",- })- }- return- }-- args := []any{- "id", crashedID,- "attempt", count,- "backoff", backoff,- }- if s.hasRestartLimit() {- args = append(args, "maxRestarts", maxRestarts)- }- slog.Warn(s.logPrefix("instance crashed, scheduling restart"), args...)-- // Emit restarting event.- if s.orch != nil {- s.orch.EmitEvent("instance.restarting", &bridge.Instance{- ID: crashedID,- Status: "restarting",- })- }-- // Wait for backoff period (respecting cancellation).- select {- case <-time.After(backoff):- case <-ctx.Done():- return- }-- s.restartInstance()-}--// restartInstance launches a new instance to replace the crashed one.-func (s *Strategy) restartInstance() {- s.mu.Lock()- ctx := s.ctx- oldID := s.instanceID- headless := s.headless- s.mu.Unlock()-- if ctx == nil || ctx.Err() != nil {- s.mu.Lock()- s.restarting = false- s.mu.Unlock()- return- }-- // Clean up the old crashed instance so the orchestrator releases the- // profile slot and allocated port before we attempt a new launch.- if oldID != "" {- if err := s.orch.Stop(oldID); err != nil {- slog.Debug(s.logPrefix("stop old instance (may already be gone)"), "id", oldID, "err", err)- }- }-- inst, err := s.orch.Launch(s.config.ProfileName, "", headless, nil)- if err != nil {- slog.Error(s.logPrefix("restart failed"),- "oldId", oldID,- "err", err,- )- s.mu.Lock()- s.restarting = false- s.mu.Unlock()- return- }-- s.mu.Lock()- s.instanceID = inst.ID- s.headless = inst.Headless- s.lastStart = time.Now()- count := s.restartCount- s.restarting = false- s.mu.Unlock()-- slog.Info(s.logPrefix("instance restarted"),- "oldId", oldID,- "newId", inst.ID,- "attempt", count,- )-- // Emit restarted event for dashboard/SSE consumers.- s.orch.EmitEvent("instance.restarted", inst)-}--// stabilityLoop resets the restart counter after the instance runs stably.-func (s *Strategy) stabilityLoop() {- ticker := time.NewTicker(30 * time.Second)- defer ticker.Stop()-- for {- s.mu.Lock()- ctx := s.ctx- s.mu.Unlock()-- if ctx == nil {- return- }-- select {- case <-ctx.Done():- return- case <-ticker.C:- s.mu.Lock()- if s.restartCount > 0 && !s.lastStart.IsZero() && time.Since(s.lastStart) > s.config.StableAfter {- slog.Info(s.logPrefix("instance stable, resetting restart counter"),- "id", s.instanceID,- "stableFor", time.Since(s.lastStart).Round(time.Second),- )- s.restartCount = 0- }- s.mu.Unlock()- }- }-}--// proxyToManaged ensures the managed instance is running, then proxies.-func (s *Strategy) proxyToManaged(w http.ResponseWriter, r *http.Request) {- target, err := s.ensureRunning()- if err != nil {- httpx.Error(w, 503, err)- return- }- activity.EnrichRouteActivity(r)- strategy.EnrichForTarget(r, s.orch, target)- s.orch.ProxyToTarget(w, r, target+r.URL.Path)-}--// ensureRunning returns the URL of the managed instance if running.-func (s *Strategy) ensureRunning() (string, error) {- if s.orch == nil {- return "", fmt.Errorf("no orchestrator configured")- }- if target := s.orch.FirstRunningURL(); target != "" {- return target, nil- }- return "", fmt.Errorf("instance not ready (may be restarting)")-}--func (s *Strategy) handleTabs(w http.ResponseWriter, r *http.Request) {- target := s.orch.FirstRunningURL()- if target == "" {- httpx.JSON(w, 200, map[string]any{"tabs": []any{}})- return- }- s.orch.ProxyToTarget(w, r, target+"/tabs")-}--func (s *Strategy) handleStatus(w http.ResponseWriter, r *http.Request) {- httpx.JSON(w, 200, s.State())-}-func (s *Strategy) hasRestartLimit() bool {return s.config.MaxRestarts > 0}diff --git a/internal/strategy/autorestart/handlers.go b/internal/strategy/autorestart/handlers.gonew file mode 100644index 00000000..192a75f2--- /dev/null+++ b/internal/strategy/autorestart/handlers.go@@ -0,0 +1,54 @@+package autorestart++import (+ "fmt"+ "net/http"++ "github.com/pinchtab/pinchtab/internal/activity"+ "github.com/pinchtab/pinchtab/internal/httpx"+ "github.com/pinchtab/pinchtab/internal/strategy"+)++// RegisterRoutes adds shorthand endpoints that proxy to the managed instance.+func (s *Strategy) RegisterRoutes(mux *http.ServeMux) {+ s.orch.RegisterHandlers(mux)+ strategy.RegisterShorthandRoutes(mux, s.orch, s.proxyToManaged)+ mux.HandleFunc("GET /tabs", s.handleTabs)+ mux.HandleFunc("GET "+s.config.StatusPath, s.handleStatus)+}++// proxyToManaged ensures the managed instance is running, then proxies.+func (s *Strategy) proxyToManaged(w http.ResponseWriter, r *http.Request) {+ target, err := s.ensureRunning()+ if err != nil {+ httpx.Error(w, 503, err)+ return+ }+ activity.EnrichRouteActivity(r)+ strategy.EnrichForTarget(r, s.orch, target)+ s.orch.ProxyToTarget(w, r, target+r.URL.Path)+}++// ensureRunning returns the URL of the managed instance if running.+func (s *Strategy) ensureRunning() (string, error) {+ if s.orch == nil {+ return "", fmt.Errorf("no orchestrator configured")+ }+ if target := s.orch.FirstRunningURL(); target != "" {+ return target, nil+ }+ return "", fmt.Errorf("instance not ready (may be restarting)")+}++func (s *Strategy) handleTabs(w http.ResponseWriter, r *http.Request) {+ target := s.orch.FirstRunningURL()+ if target == "" {+ httpx.JSON(w, 200, map[string]any{"tabs": []any{}})+ return+ }+ s.orch.ProxyToTarget(w, r, target+"/tabs")+}++func (s *Strategy) handleStatus(w http.ResponseWriter, r *http.Request) {+ httpx.JSON(w, 200, s.State())+}diff --git a/internal/strategy/autorestart/restart.go b/internal/strategy/autorestart/restart.gonew file mode 100644index 00000000..5552ac2e--- /dev/null+++ b/internal/strategy/autorestart/restart.go@@ -0,0 +1,220 @@+package autorestart++import (+ "context"+ "log/slog"+ "time"++ "github.com/pinchtab/pinchtab/internal/bridge"+ "github.com/pinchtab/pinchtab/internal/orchestrator"+)++// launchInitial launches the first instance (or after strategy start).+func (s *Strategy) launchInitial() {+ s.mu.Lock()+ ctx := s.ctx+ s.mu.Unlock()++ if ctx == nil || ctx.Err() != nil {+ return+ }++ inst, err := s.orch.Launch(s.config.ProfileName, "", s.config.Headless, nil)+ if err != nil {+ slog.Error(s.logPrefix("initial launch failed"), "profile", s.config.ProfileName, "err", err)+ return+ }++ s.mu.Lock()+ s.instanceID = inst.ID+ s.headless = inst.Headless+ s.lastStart = time.Now()+ s.mu.Unlock()++ slog.Info(s.logPrefix("instance launched"), "id", inst.ID, "profile", s.config.ProfileName)+}++// handleEvent processes orchestrator lifecycle events.+func (s *Strategy) handleEvent(evt orchestrator.InstanceEvent) {+ s.mu.Lock()+ managedID := s.instanceID+ deliberate := s.deliberate+ restarting := s.restarting+ ctx := s.ctx+ s.mu.Unlock()++ // Only handle events for the managed instance.+ if evt.Instance == nil || evt.Instance.ID != managedID {+ return+ }++ // Skip if a restart is already in progress (prevents duplicate handling+ // when both instance.error and instance.stopped fire for the same crash).+ if restarting {+ return+ }++ switch evt.Type {+ case "instance.stopped":+ if deliberate {+ slog.Info(s.logPrefix("instance stopped deliberately"), "id", managedID)+ return+ }+ // Instance exited unexpectedly — check if we should restart.+ s.handleCrash(ctx, managedID)++ case "instance.error":+ if deliberate {+ return+ }+ s.handleCrash(ctx, managedID)+ }+}++// handleCrash decides whether to restart the crashed instance.+func (s *Strategy) handleCrash(ctx context.Context, crashedID string) {+ if ctx == nil || ctx.Err() != nil {+ return+ }++ s.mu.Lock()+ s.restarting = true+ s.restartCount+++ s.lastCrash = time.Now()+ count := s.restartCount+ maxRestarts := s.config.MaxRestarts+ backoff := s.config.InitBackoff * time.Duration(1<<uint(count-1))+ if backoff > s.config.MaxBackoff {+ backoff = s.config.MaxBackoff+ }+ s.mu.Unlock()++ if s.hasRestartLimit() && count > maxRestarts {+ slog.Error(s.logPrefix("max restarts exceeded, giving up"),+ "id", crashedID,+ "restartCount", count-1,+ "maxRestarts", maxRestarts,+ )+ s.mu.Lock()+ s.restarting = false+ s.mu.Unlock()+ if s.orch != nil {+ s.orch.EmitEvent("instance.crashed", &bridge.Instance{+ ID: crashedID,+ Status: "crashed",+ })+ }+ return+ }++ args := []any{+ "id", crashedID,+ "attempt", count,+ "backoff", backoff,+ }+ if s.hasRestartLimit() {+ args = append(args, "maxRestarts", maxRestarts)+ }+ slog.Warn(s.logPrefix("instance crashed, scheduling restart"), args...)++ // Emit restarting event.+ if s.orch != nil {+ s.orch.EmitEvent("instance.restarting", &bridge.Instance{+ ID: crashedID,+ Status: "restarting",+ })+ }++ // Wait for backoff period (respecting cancellation).+ select {+ case <-time.After(backoff):+ case <-ctx.Done():+ return+ }++ s.restartInstance()+}++// restartInstance launches a new instance to replace the crashed one.+func (s *Strategy) restartInstance() {+ s.mu.Lock()+ ctx := s.ctx+ oldID := s.instanceID+ headless := s.headless+ s.mu.Unlock()++ if ctx == nil || ctx.Err() != nil {+ s.mu.Lock()+ s.restarting = false+ s.mu.Unlock()+ return+ }++ // Clean up the old crashed instance so the orchestrator releases the+ // profile slot and allocated port before we attempt a new launch.+ if oldID != "" {+ if err := s.orch.Stop(oldID); err != nil {+ slog.Debug(s.logPrefix("stop old instance (may already be gone)"), "id", oldID, "err", err)+ }+ }++ inst, err := s.orch.Launch(s.config.ProfileName, "", headless, nil)+ if err != nil {+ slog.Error(s.logPrefix("restart failed"),+ "oldId", oldID,+ "err", err,+ )+ s.mu.Lock()+ s.restarting = false+ s.mu.Unlock()+ return+ }++ s.mu.Lock()+ s.instanceID = inst.ID+ s.headless = inst.Headless+ s.lastStart = time.Now()+ count := s.restartCount+ s.restarting = false+ s.mu.Unlock()++ slog.Info(s.logPrefix("instance restarted"),+ "oldId", oldID,+ "newId", inst.ID,+ "attempt", count,+ )++ // Emit restarted event for dashboard/SSE consumers.+ s.orch.EmitEvent("instance.restarted", inst)+}++// stabilityLoop resets the restart counter after the instance runs stably.+func (s *Strategy) stabilityLoop() {+ ticker := time.NewTicker(30 * time.Second)+ defer ticker.Stop()++ for {+ s.mu.Lock()+ ctx := s.ctx+ s.mu.Unlock()++ if ctx == nil {+ return+ }++ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ s.mu.Lock()+ if s.restartCount > 0 && !s.lastStart.IsZero() && time.Since(s.lastStart) > s.config.StableAfter {+ slog.Info(s.logPrefix("instance stable, resetting restart counter"),+ "id", s.instanceID,+ "stableFor", time.Since(s.lastStart).Round(time.Second),+ )+ s.restartCount = 0+ }+ s.mu.Unlock()+ }+ }+}diff --git a/internal/strategy/routes.go b/internal/strategy/routes.goindex e3e66219..8cf04763 100644--- a/internal/strategy/routes.go+++ b/internal/strategy/routes.go@@ -7,26 +7,6 @@ import ("github.com/pinchtab/pinchtab/internal/routes")-// capabilityEnabled maps a routes.Capability to the orchestrator check.-func capabilityEnabled(orch *orchestrator.Orchestrator, cap routes.Capability) bool {- switch cap {- case routes.CapEvaluate:- return orch.AllowsEvaluate()- case routes.CapMacro:- return orch.AllowsMacro()- case routes.CapScreencast:- return orch.AllowsScreencast()- case routes.CapDownload:- return orch.AllowsDownload()- case routes.CapUpload:- return orch.AllowsUpload()- case routes.CapStateExport:- return orch.AllowsStateExport()- default:- return false- }-}-// capabilitySetting returns the config key and disabled code for a capability.func capabilitySetting(cap routes.Capability) (feature, setting, code string) {switch cap {@@ -59,7 +39,7 @@ func RegisterShorthandRoutes(mux *http.ServeMux, orch *orchestrator.Orchestrator}for cap, eps := range routes.CapabilityEndpoints() {- enabled := capabilityEnabled(orch, cap)+ enabled := orch.Allows(cap)feature, setting, code := capabilitySetting(cap)for _, ep := range eps {RegisterCapabilityRoute(mux, ep.Route(), enabled, feature, setting, code, handler)