Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

19.9 KB
Raw Blame
// profile_lock.go handles stale browser profile lock recovery for
// Chromium-based providers.
//
// When a container restarts (or the browser crashes), Chromium's
// SingletonLock, SingletonSocket, and SingletonCookie files may be left behind
// in the profile directory. On next startup the browser sees these and refuses
// to launch with
// "The profile appears to be in use by another Chromium process".
//
// This code detects that error, checks whether the owning process is actually
// still running (via PID probe and process listing), and removes the stale
// lock files if it's safe to do so. It retries browser startup once after
// clearing the locks.

package bridge

import (
	"bytes"
	"fmt"
	"io/fs"
	"log/slog"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

var chromeProfileProcessLister = findChromeProfileProcesses
var chromePIDIsRunning = isChromePIDRunning
var killChromeProfileProcesses = killProcesses
var isProfileOwnedByRunningPinchtabMock = isProfileOwnedByRunningPinchtab
var isPinchTabProcessFunc = isPinchTabProcess

// lockStartupGrace is the window after a PID file is written during which the
// browser process check is skipped. AcquireProfileLock writes the PID before
// InitBrowser launches the browser, so a second process must not treat the
// missing browser as a stale lock during this startup window.
var lockStartupGrace = 2 * time.Minute

var chromeSingletonFiles = []string{
	"SingletonLock",
	"SingletonSocket",
	"SingletonCookie",
}

var chromeProfileLockPIDPattern = regexp.MustCompile(`(?:Chromium|Chrome) process \((\d+)\)`)

type chromeProfileProcess struct {
	PID     string
	Command string
}

func isProfileLockError(msg string) bool {
	if msg == "" {
		return false
	}
	return strings.Contains(msg, "The profile appears to be in use by another Chromium process") ||
		strings.Contains(msg, "The profile appears to be in use by another Chrome process") ||
		strings.Contains(msg, "process_singleton")
}

func clearStaleProfileLocks(profileDir, errMsg string) (bool, error) {
	if strings.TrimSpace(profileDir) == "" {
		return false, nil
	}

	if pid, ok := extractChromeProfileLockPID(errMsg); ok {
		running, err := chromePIDIsRunning(pid)
		if err != nil {
			slog.Warn("failed to probe browser lock pid; falling back to process listing", "profile", profileDir, "pid", pid, "err", err)
		} else if running {
			if owned, ptPid := isProfileOwnedByRunningPinchtabMock(profileDir); owned {
				slog.Warn("browser profile lock appears active and owned by another pinchtab; leaving singleton files in place", "profile", profileDir, "pid", pid, "pinchtab_pid", ptPid)
				return false, nil
			}
			slog.Warn("browser profile lock appears active but pinchtab owner is dead; proceeding with stale cleanup", "profile", profileDir, "pid", pid)
		}
	}

	processes, err := chromeProfileProcessLister(profileDir)
	if err != nil {
		if _, ok := extractChromeProfileLockPID(errMsg); ok {
			slog.Warn("profile process listing unavailable; proceeding with stale lock cleanup based on lock pid", "profile", profileDir, "err", err)
		} else {
			return false, err
		}
	}
	if len(processes) > 0 {
		if owned, ptPid := isProfileOwnedByRunningPinchtabMock(profileDir); owned {
			pids := make([]string, 0, len(processes))
			for _, proc := range processes {
				pids = append(pids, proc.PID)
			}
			slog.Warn("browser profile lock appears active and owned by another pinchtab; leaving singleton files in place", "profile", profileDir, "pids", strings.Join(pids, ","), "pinchtab_pid", ptPid)
			return false, nil
		}

		slog.Warn("browser profile lock appears active but no pinchtab owner found; killing stale processes", "profile", profileDir)
		if err := killChromeProfileProcesses(processes); err != nil {
			slog.Error("failed to kill stale browser processes", "profile", profileDir, "err", err)
			return false, nil
		}
	}

	removed := false
	for _, name := range chromeSingletonFiles {
		path := filepath.Join(profileDir, name)
		if _, err := os.Lstat(path); err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return false, fmt.Errorf("inspect %s: %w", path, err)
		}
		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
			return removed, fmt.Errorf("remove %s: %w", path, err)
		}
		removed = true
	}

	return removed, nil
}

// quarantineExitWait bounds how long quarantine waits for the dying browser
// to release the profile before renaming. Var so tests can shrink it.
var quarantineExitWait = 5 * time.Second

const quarantineSuffix = ".quarantine-"

var quarantineDirName = regexp.MustCompile(`^(.+)` + regexp.QuoteMeta(quarantineSuffix) + `(\d+)$`)

// SplitQuarantineName reads a quarantine directory name back into the profile it was
// made from and the timestamp it carries. It is the one owner of the pattern: the
// predicate below and the prune both go through it, so a reader cannot drift from what
// quarantine writes.
func SplitQuarantineName(dirName string) (profile string, stamp int64, ok bool) {
	match := quarantineDirName.FindStringSubmatch(dirName)
	if match == nil {
		return "", 0, false
	}
	stamp, err := strconv.ParseInt(match[2], 10, 64)
	if err != nil {
		return "", 0, false
	}
	return match[1], stamp, true
}

func IsQuarantinedProfileDir(dirName string) bool {
	_, _, ok := SplitQuarantineName(dirName)
	return ok
}

type QuarantineRemoval struct {
	Path  string
	Bytes int64
}

const KeepAllQuarantinedProfiles = 0

// PruneQuarantinedProfiles keeps the newest `keep` quarantined siblings of one profile
// and removes the rest, returning what it reclaimed. It carries the AUTOMATIC retention
// policy — a keep count of zero or less means keep everything — which is why the
// on-demand reclaim below does not go through it: there is no keep value that means
// "remove them all", so remove-all is expressed by asking removeQuarantinedProfiles for
// an unranked selection rather than by a sentinel keep count. Both end at the same
// deleter.
//
// It is not the only code that can remove a quarantine directory: DELETE /profiles/{id}
// resolves an id through the profile listing — where a quarantine appears under its own
// id — and calls ProfileManager.Delete, which removes the directory by name with neither
// guard. That route is authenticated and audit-logged and is arguably the reclaim a user
// wants, so the distinction to preserve is which deleter is scoped, not which one exists.
// TestOnlyOneFunctionRemovesADirectoryUnderTheProfilesBase is the census of that set.
//
// A user who names a profile exactly "<other profile>.quarantine-<digits>" is
// indistinguishable on disk from a real quarantine; nothing here can tell them apart,
// and what bounds the exposure is the sibling scope. See
// TestPruneOnlyReachesALookalikeThroughItsNamesakeAndStillKeepsTheNewest.
//
// justCreated is excluded by PATH, not by being newest: quarantine may proceed while a
// dying browser still holds the directory, so the one entry that can still be written to
// must survive a same-second timestamp tie.
func PruneQuarantinedProfiles(profileDir, justCreated string, keep int) ([]QuarantineRemoval, error) {
	if keep <= KeepAllQuarantinedProfiles {
		return nil, nil
	}
	profileDir = strings.TrimSpace(profileDir)
	if profileDir == "" {
		return nil, nil
	}

	siblings, err := quarantinedSiblings(profileDir)
	if err != nil {
		return nil, err
	}
	return removeQuarantinedProfiles(siblingsBeyondKeep(siblings, justCreated, keep)), nil
}

// siblingsBeyondKeep ranks one profile's quarantined siblings newest first and names the
// ones the keep count does not cover. Ranking is separated from the deletion so the
// on-demand reclaim can reach the same deleter with no ranking at all.
func siblingsBeyondKeep(siblings []quarantinedSibling, justCreated string, keep int) []string {
	sort.Slice(siblings, func(i, j int) bool { return siblings[i].stamp > siblings[j].stamp })

	// justCreated reserves a slot rather than being ranked, which is what makes the
	// outcome independent of the timestamp order.
	budget := keep
	for _, sibling := range siblings {
		if sibling.path == justCreated {
			budget--
			break
		}
	}

	var doomed []string
	for _, sibling := range siblings {
		if sibling.path == justCreated {
			continue
		}
		if budget > 0 {
			budget--
			continue
		}
		doomed = append(doomed, sibling.path)
	}
	return doomed
}

// removeQuarantinedProfiles is the one function that deletes a quarantined profile
// directory. It re-applies IsQuarantinedProfileDir to every path it is handed rather
// than trusting its caller to have filtered: the callers differ in how they select
// (ranked siblings of one profile, or every quarantine under the base), and a live
// profile must be unreachable however the selection is shaped, so the guard belongs
// where the deletion is.
func removeQuarantinedProfiles(paths []string) []QuarantineRemoval {
	var removals []QuarantineRemoval
	for _, path := range paths {
		if !IsQuarantinedProfileDir(filepath.Base(path)) {
			slog.Warn("refused to remove a profile that is not quarantined", "profile", path)
			continue
		}
		reclaimed := dirBytes(path)
		if err := os.RemoveAll(path); err != nil {
			slog.Warn("could not remove quarantined profile", "profile", path, "err", err)
			continue
		}
		slog.Info("removed quarantined profile", "profile", path, "bytes", reclaimed)
		removals = append(removals, QuarantineRemoval{Path: path, Bytes: reclaimed})
	}
	return removals
}

// ReclaimableQuarantinedProfiles reports what a reclaim would remove and what it would
// free, removing nothing. It is the dry run the bare command depends on.
func ReclaimableQuarantinedProfiles(baseDir, only string) ([]QuarantineRemoval, error) {
	paths, err := quarantinedProfileSelection(baseDir, only)
	if err != nil {
		return nil, err
	}
	reclaimable := make([]QuarantineRemoval, 0, len(paths))
	for _, path := range paths {
		reclaimable = append(reclaimable, QuarantineRemoval{Path: path, Bytes: dirBytes(path)})
	}
	return reclaimable, nil
}

// ReclaimQuarantinedProfiles removes the quarantined directories a reclaim selects,
// on demand and with no retention policy: this is the half PruneQuarantinedProfiles
// cannot express, and it stays reachable when quarantineKeep is set to keep everything.
func ReclaimQuarantinedProfiles(baseDir, only string) ([]QuarantineRemoval, error) {
	paths, err := quarantinedProfileSelection(baseDir, only)
	if err != nil {
		return nil, err
	}
	return removeQuarantinedProfiles(paths), nil
}

// quarantinedProfileSelection names the quarantined directories directly under baseDir,
// optionally narrowed to one of them.
//
// `only` is matched against the names of directories already enumerated and already
// found quarantined; it is never joined onto baseDir. That is what makes a traversal
// impossible rather than merely refused — caller text cannot become a path component
// here, so no amount of "../" reaches outside the profile root. The explicit refusal
// below exists so a caller that sends a path gets told that, instead of a bare not-found.
func quarantinedProfileSelection(baseDir, only string) ([]string, error) {
	baseDir = strings.TrimSpace(baseDir)
	if baseDir == "" {
		return nil, fmt.Errorf("no profiles directory configured")
	}
	only = strings.TrimSpace(only)
	if only != "" && only != filepath.Base(only) {
		return nil, fmt.Errorf("%q is a path, not a quarantined profile name", only)
	}

	entries, err := os.ReadDir(baseDir)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("read profiles dir: %w", err)
	}

	var paths []string
	for _, entry := range entries {
		if !entry.IsDir() || !IsQuarantinedProfileDir(entry.Name()) {
			continue
		}
		if only != "" && entry.Name() != only {
			continue
		}
		paths = append(paths, filepath.Join(baseDir, entry.Name()))
	}
	if only != "" && len(paths) == 0 {
		return nil, fmt.Errorf("no quarantined profile named %q", only)
	}
	return paths, nil
}

type quarantinedSibling struct {
	path  string
	stamp int64
}

// quarantinedSiblings finds the quarantined directories belonging to one profile. Both
// halves come from SplitQuarantineName — whether a name is a quarantine at all, and whose
// it is — so neither rule has a second implementation that could drift.
func quarantinedSiblings(profileDir string) ([]quarantinedSibling, error) {
	parent := filepath.Dir(profileDir)
	profileName := filepath.Base(profileDir)
	entries, err := os.ReadDir(parent)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("read profiles dir: %w", err)
	}

	var siblings []quarantinedSibling
	for _, entry := range entries {
		if !entry.IsDir() {
			continue
		}
		profile, stamp, ok := SplitQuarantineName(entry.Name())
		if !ok {
			continue
		}
		if profile != profileName {
			continue
		}
		siblings = append(siblings, quarantinedSibling{path: filepath.Join(parent, entry.Name()), stamp: stamp})
	}
	return siblings, nil
}

func dirBytes(path string) int64 {
	var total int64
	_ = filepath.WalkDir(path, func(_ string, entry fs.DirEntry, err error) error {
		if err != nil || entry.IsDir() {
			return nil
		}
		if info, err := entry.Info(); err == nil {
			total += info.Size()
		}
		return nil
	})
	return total
}

// quarantineCorruptedProfile renames profileDir to "<profileDir>.quarantine-<ts>"
// and recreates an empty directory at the original path. Used to recover
// from silent CDP attach failures where CloakBrowser refuses to ingest
// existing profile state.
//
// keep bounds how many quarantined siblings of this profile survive, newest first,
// pruned here rather than on a startup sweep: a sweep would delete directories the
// operator never asked about, while this only reclaims as the same profile keeps
// failing. Directories belonging to profiles that never quarantine again are
// therefore never reclaimed by this path.
func quarantineCorruptedProfile(profileDir string, keep int) (string, error) {
	profileDir = strings.TrimSpace(profileDir)
	if profileDir == "" {
		return "", fmt.Errorf("empty profile dir")
	}
	if _, err := os.Stat(profileDir); err != nil {
		if os.IsNotExist(err) {
			return "", nil
		}
		return "", fmt.Errorf("stat profile dir: %w", err)
	}
	// The caller cancels chromedp contexts without waiting for the process to
	// exit; renaming under a still-running browser fails on Windows and lets
	// the old process keep writing into the quarantined dir on POSIX. On
	// timeout, proceed anyway — refusing entirely would block recovery.
	if !waitForChromeExit(profileDir, quarantineExitWait) {
		slog.Warn("quarantining profile while a browser process may still hold it", "profile", profileDir)
	}
	quarantinePath := fmt.Sprintf("%s%s%d", profileDir, quarantineSuffix, time.Now().Unix())
	if err := os.Rename(profileDir, quarantinePath); err != nil {
		return "", fmt.Errorf("rename profile dir: %w", err)
	}
	// The rename carries profile.json along, where it now names a profile this
	// directory no longer is. Drop it so nothing on disk makes that claim.
	if err := os.Remove(filepath.Join(quarantinePath, "profile.json")); err != nil && !os.IsNotExist(err) {
		slog.Warn("stale profile metadata left in quarantined profile", "profile", quarantinePath, "err", err)
	}
	if err := os.MkdirAll(profileDir, 0700); err != nil {
		return quarantinePath, fmt.Errorf("recreate profile dir: %w", err)
	}
	// After the rename, so a failed quarantine never costs an older artefact.
	removals, err := PruneQuarantinedProfiles(profileDir, quarantinePath, keep)
	if err != nil {
		slog.Warn("could not prune older quarantined profiles", "profile", profileDir, "err", err)
	}
	for _, removal := range removals {
		slog.Info("pruned older quarantined profile", "profile", removal.Path, "bytesReclaimed", removal.Bytes, "keep", keep)
	}
	return quarantinePath, nil
}

// ProfileOwnedByRunningPinchtab reports whether a live pinchtab process holds
// profileDir via its pinchtab.pid lock. It is the per-directory truth the
// profiles destructive-route guard falls back to on surfaces that have no
// orchestrator instance mapping.
func ProfileOwnedByRunningPinchtab(profileDir string) (bool, int) {
	return isProfileOwnedByRunningPinchtab(profileDir)
}

func isProfileOwnedByRunningPinchtab(profileDir string) (bool, int) {
	pidFile := filepath.Join(profileDir, "pinchtab.pid")
	data, err := os.ReadFile(pidFile)
	if err != nil {
		if !os.IsNotExist(err) {
			slog.Debug("failed to read pinchtab pid file", "path", pidFile, "err", err)
		}
		return false, 0
	}

	var pid int
	if _, err := fmt.Sscanf(string(data), "%d", &pid); err != nil {
		slog.Debug("failed to parse pinchtab pid file", "path", pidFile, "err", err)
		return false, 0
	}

	if pid == os.Getpid() {
		return false, pid // It's us
	}

	running, err := chromePIDIsRunning(pid)
	if err == nil && running {
		// Even if the PID is running, check if it's actually a pinchtab process
		// to handle PID reuse.
		if isPinchTabProcessFunc(pid) {
			// Skip the browser check while the PID file is fresh: AcquireProfileLock
			// writes the file before InitBrowser launches the browser, so a second
			// process must not steal the lock during that startup window.
			if info, statErr := os.Stat(pidFile); statErr == nil && time.Since(info.ModTime()) < lockStartupGrace {
				slog.Debug("profile lock belongs to a recently started pinchtab; treating as active",
					"profile", profileDir, "pid", pid)
				return true, pid
			}
			processes, procErr := chromeProfileProcessLister(profileDir)
			if procErr == nil && len(processes) == 0 {
				slog.Debug("pinchtab pid file points to a running pinchtab but no browser is using the profile; treating lock as stale",
					"profile", profileDir, "pid", pid)
				return false, 0
			}
			if procErr != nil {
				slog.Debug("unable to verify browser ownership for running pinchtab pid; keeping profile locked",
					"profile", profileDir, "pid", pid, "err", procErr)
			}
			slog.Debug("profile is owned by another active pinchtab", "profile", profileDir, "pid", pid)
			return true, pid
		}
		slog.Debug("PID in lockfile is running but not a pinchtab process (PID reuse)", "profile", profileDir, "pid", pid)
	} else {
		slog.Debug("PID in lockfile is not running", "profile", profileDir, "pid", pid, "err", err)
	}

	return false, 0
}

func AcquireProfileLock(profileDir string) error {
	if profileDir == "" {
		return nil
	}
	if err := os.MkdirAll(profileDir, 0755); err != nil {
		return fmt.Errorf("mkdir profile dir: %w", err)
	}

	if owned, pid := isProfileOwnedByRunningPinchtab(profileDir); owned {
		return fmt.Errorf("profile %s is already in use by pinchtab process %d", profileDir, pid)
	}

	pidFile := filepath.Join(profileDir, "pinchtab.pid")
	slog.Debug("acquiring profile lock", "profile", profileDir, "pid", os.Getpid())
	return os.WriteFile(pidFile, []byte(fmt.Sprintf("%d", os.Getpid())), 0644)
}

func extractChromeProfileLockPID(msg string) (int, bool) {
	if msg == "" {
		return 0, false
	}
	match := chromeProfileLockPIDPattern.FindStringSubmatch(msg)
	if len(match) != 2 {
		return 0, false
	}
	pid := 0
	for _, ch := range match[1] {
		pid = pid*10 + int(ch-'0')
	}
	if pid <= 0 {
		return 0, false
	}
	return pid, true
}

func findChromeProfileProcesses(profileDir string) ([]chromeProfileProcess, error) {
	if strings.TrimSpace(profileDir) == "" {
		return nil, nil
	}

	cmd := exec.Command("ps", "-axo", "pid=,args=")
	out, err := cmd.Output()
	if err != nil {
		return nil, fmt.Errorf("list chrome processes: %w", err)
	}

	return parseChromeProfileProcesses(out, profileDir), nil
}

func parseChromeProfileProcesses(out []byte, profileDir string) []chromeProfileProcess {
	if len(out) == 0 || strings.TrimSpace(profileDir) == "" {
		return nil
	}

	needleEquals := "--user-data-dir=" + profileDir
	needleSpace := "--user-data-dir " + profileDir
	lines := bytes.Split(out, []byte{'\n'})
	processes := make([]chromeProfileProcess, 0)

	for _, rawLine := range lines {
		line := strings.TrimSpace(string(rawLine))
		if line == "" || (!strings.Contains(line, needleEquals) && !strings.Contains(line, needleSpace)) {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) < 2 {
			continue
		}

		processes = append(processes, chromeProfileProcess{
			PID:     fields[0],
			Command: strings.TrimSpace(strings.TrimPrefix(line, fields[0])),
		})
	}

	return processes
}