Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

9.2 KB
Raw Blame
package orchestrator

import (
	"net/http"
	"net/http/httptest"
	"sync"
	"testing"
	"time"

	"github.com/pinchtab/pinchtab/internal/activity"
	"github.com/pinchtab/pinchtab/internal/bridge"
)

const stubInstanceID = "inst_probe"

func hangUntilClientGivesUp(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() }

func answerHealth(w http.ResponseWriter, _ *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	_, _ = w.Write([]byte(`{"status":"ok","tabs":0}`))
}

func answerTabs(w http.ResponseWriter, _ *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	_, _ = w.Write([]byte(`{"tabs":[]}`))
}

func shortProbeBudget(t *testing.T) time.Duration {
	t.Helper()
	previous := responsivenessProbeBudget
	responsivenessProbeBudget = 150 * time.Millisecond
	t.Cleanup(func() { responsivenessProbeBudget = previous })
	return responsivenessProbeBudget
}

func orchestratorOverStubChild(t *testing.T, handler http.Handler) (*Orchestrator, *mockRunner) {
	t.Helper()
	backend := httptest.NewServer(handler)
	t.Cleanup(backend.Close)
	runner := &mockRunner{portAvail: true}
	o := NewOrchestratorWithRunner(t.TempDir(), runner)
	o.client = backend.Client()
	o.mu.Lock()
	o.instances[stubInstanceID] = &InstanceInternal{
		Instance: bridge.Instance{ID: stubInstanceID, ProfileName: "p1", Status: "running"},
		URL:      backend.URL,
	}
	o.mu.Unlock()
	return o, runner
}

func stubChild(health, tabs http.HandlerFunc) http.Handler {
	mux := http.NewServeMux()
	mux.HandleFunc("/health", health)
	mux.HandleFunc("/tabs", tabs)
	return mux
}

func TestResponsivenessIsThreeDistinctValuesNeverSharingARepresentation(t *testing.T) {
	values := map[string]bool{
		bridge.ResponsivenessResponsive:   true,
		bridge.ResponsivenessUnresponsive: true,
		bridge.ResponsivenessUnknown:      true,
	}
	if len(values) != 3 {
		t.Fatalf("responsiveness values collide: %v", values)
	}
	for _, v := range []string{"", "garbage"} {
		if got := bridge.NormalizeResponsiveness(v); got != bridge.ResponsivenessUnknown {
			t.Fatalf("NormalizeResponsiveness(%q) = %q, want unknown", v, got)
		}
	}
}

func TestAnInstanceAnsweringHealthButNotTabsIsUnresponsiveAndStillRunning(t *testing.T) {
	shortProbeBudget(t)
	o, runner := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))
	o.mu.RLock()
	before := o.instances[stubInstanceID]
	o.mu.RUnlock()

	if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {
		t.Fatalf("never-probed instance reads %q, want unknown", got)
	}
	o.RefreshCrashes()

	inst := o.List()[0]
	if inst.Responsiveness != bridge.ResponsivenessUnresponsive {
		t.Fatalf("responsiveness = %q, want unresponsive", inst.Responsiveness)
	}
	if inst.Status != "running" {
		t.Fatalf("status = %q, want running: responsiveness never changes routing", inst.Status)
	}
	o.mu.RLock()
	after := o.instances[stubInstanceID]
	o.mu.RUnlock()
	if runner.runCalled || after != before {
		t.Fatalf("an unresponsive result restarted the instance (runCalled=%v, replaced=%v)", runner.runCalled, after != before)
	}
}

func TestAnInstanceBlockingEveryRouteIsUnknownNotUnresponsive(t *testing.T) {
	shortProbeBudget(t)
	o, _ := orchestratorOverStubChild(t, stubChild(hangUntilClientGivesUp, hangUntilClientGivesUp))
	o.RefreshCrashes()
	if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {
		t.Fatalf("responsiveness = %q, want unknown: its /health never answered either", got)
	}
}

func TestAnInstanceAnsweringBothRoutesIsResponsive(t *testing.T) {
	shortProbeBudget(t)
	o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, answerTabs))
	o.RefreshCrashes()
	if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessResponsive {
		t.Fatalf("responsiveness = %q, want responsive", got)
	}
}

func TestAStoppedInstanceReadsUnknownAgainAfterTheNextRefresh(t *testing.T) {
	shortProbeBudget(t)
	o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, answerTabs))
	o.RefreshCrashes()
	o.mu.Lock()
	o.instances[stubInstanceID].Status = "stopped"
	o.mu.Unlock()
	o.RefreshCrashes()
	if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {
		t.Fatalf("responsiveness = %q after the instance stopped, want unknown", got)
	}
}

func TestTheProbeBudgetBoundsTheCrashSummaryLatency(t *testing.T) {
	budget := shortProbeBudget(t)
	o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))
	started := time.Now()
	o.CrashSummary()
	elapsed := time.Since(started)
	if elapsed < budget || elapsed > budget+500*time.Millisecond {
		t.Fatalf("CrashSummary took %v with one hanging child, want within the %v probe budget plus a small margin", elapsed, budget)
	}
}

type capturingRecorder struct {
	mu     sync.Mutex
	events []activity.Event
}

func (c *capturingRecorder) Enabled() bool { return true }
func (c *capturingRecorder) Record(evt activity.Event) error {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.events = append(c.events, evt)
	return nil
}
func (c *capturingRecorder) Query(activity.Filter) ([]activity.Event, error) { return nil, nil }

func TestTheTabsProbeReachesTheChildAsOrchestratorActivityNotClientActivity(t *testing.T) {
	shortProbeBudget(t)
	rec := &capturingRecorder{}
	o, _ := orchestratorOverStubChild(t, activity.Middleware(rec, "client", stubChild(answerHealth, answerTabs)))
	o.RefreshCrashes()

	rec.mu.Lock()
	defer rec.mu.Unlock()
	paths := map[string]string{}
	for _, evt := range rec.events {
		paths[evt.Path] = evt.Source
	}
	for _, path := range []string{"/tabs", "/health"} {
		if source, ok := paths[path]; !ok || source != orchestratorActivitySource {
			t.Fatalf("probe of %s recorded with source %q (seen=%v), want %q so it counts as neither activity nor an idle reset", path, source, ok, orchestratorActivitySource)
		}
	}
}

func awaitRefresh(t *testing.T, o *Orchestrator) time.Time {
	t.Helper()
	o.refreshMu.Lock()
	done := o.refreshDone
	o.refreshMu.Unlock()
	if done != nil {
		select {
		case <-done:
		case <-time.After(5 * time.Second):
			t.Fatal("the crash fan-out never finished")
		}
	}
	return time.Now()
}

func addStubInstance(o *Orchestrator, id, url string) {
	o.mu.Lock()
	defer o.mu.Unlock()
	o.instances[id] = &InstanceInternal{
		Instance: bridge.Instance{ID: id, ProfileName: id, Status: "running"},
		URL:      url,
	}
}

func TestAHangingTabsProbeReleasesTheRequestAtTheJoinWaitAndRecordsItsVerdictLater(t *testing.T) {
	previous := responsivenessProbeBudget
	responsivenessProbeBudget = 800 * time.Millisecond
	t.Cleanup(func() { responsivenessProbeBudget = previous })
	o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))

	started := time.Now()
	o.CrashSummary()
	elapsed := time.Since(started)
	if elapsed < refreshJoinWait || elapsed > refreshJoinWait+150*time.Millisecond || elapsed >= 500*time.Millisecond {
		t.Fatalf("CrashSummary took %v with one hanging child, want the %v join wait, under the CLI's 500ms presence check", elapsed, refreshJoinWait)
	}
	if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {
		t.Fatalf("responsiveness = %q before the probe finished, want unknown", got)
	}
	awaitRefresh(t, o)
	if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnresponsive {
		t.Fatalf("responsiveness = %q after the probe finished, want unresponsive", got)
	}
}

func TestConcurrentRefreshesShareOneFanOut(t *testing.T) {
	previous := responsivenessProbeBudget
	responsivenessProbeBudget = 800 * time.Millisecond
	t.Cleanup(func() { responsivenessProbeBudget = previous })
	var mu sync.Mutex
	tabsHits := 0
	o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, func(w http.ResponseWriter, r *http.Request) {
		mu.Lock()
		tabsHits++
		mu.Unlock()
		hangUntilClientGivesUp(w, r)
	}))

	o.RefreshCrashes()
	o.RefreshCrashes()
	awaitRefresh(t, o)
	mu.Lock()
	defer mu.Unlock()
	if tabsHits != 1 {
		t.Fatalf("two refreshes during one in-flight fan-out probed /tabs %d times, want 1", tabsHits)
	}
}

func TestTheFanOutProbesInstancesInParallel(t *testing.T) {
	budget := 300 * time.Millisecond
	previous := responsivenessProbeBudget
	responsivenessProbeBudget = budget
	t.Cleanup(func() { responsivenessProbeBudget = previous })
	o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))
	o.mu.RLock()
	url := o.instances[stubInstanceID].URL
	o.mu.RUnlock()
	addStubInstance(o, "inst_probe_2", url)

	started := time.Now()
	o.RefreshCrashes()
	elapsed := awaitRefresh(t, o).Sub(started)
	if elapsed > budget+150*time.Millisecond {
		t.Fatalf("fan-out over two hanging instances took %v, want one %v budget: the probes ran serially", elapsed, budget)
	}
	for _, inst := range o.List() {
		if inst.Responsiveness != bridge.ResponsivenessUnresponsive {
			t.Fatalf("%s responsiveness = %q, want unresponsive", inst.ID, inst.Responsiveness)
		}
	}
}

func TestATabsProbeCutOffWithoutAnAnswerIsUnknownNotUnresponsive(t *testing.T) {
	shortProbeBudget(t)
	o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, func(w http.ResponseWriter, _ *http.Request) {
		conn, _, err := w.(http.Hijacker).Hijack()
		if err == nil {
			_ = conn.Close()
		}
	}))
	o.RefreshCrashes()
	awaitRefresh(t, o)
	if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {
		t.Fatalf("responsiveness = %q when /tabs dropped the connection, want unknown: only a probe that outlasts the budget is unresponsive", got)
	}
}