Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

16.1 KB
Raw Blame
// Package d1 hosts WebBrain's on-device decision model
// (webbrain-one/d1-browser-decision-fp32) natively: it downloads the pinned
// ONNX graphs and an ONNX Runtime build, then answers the same
// {state, images, questions} requests as the extension's WebGPU adapter.
package d1

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/esokullu/webbrain-systray/internal/download"
	"github.com/esokullu/webbrain-systray/internal/logx"
	"github.com/esokullu/webbrain-systray/internal/paths"
	"github.com/esokullu/webbrain-systray/internal/settings"
)

const (
	ModelName = "d1-browser-decision-fp32"
	Provider  = "companion_d1"
	repo      = "webbrain-one/d1-browser-decision-fp32"
	revision  = "9023fa407e3f17e02ad1fba1b921a0dc846dca89"

	ortVersion = "1.31.0"

	PhaseIdle        = "idle"
	PhaseDownloading = "downloading"
	PhaseInstalling  = "installing"
	PhaseLoading     = "loading"
	PhaseReady       = "ready"
	PhaseError       = "error"
)

// ErrNotReady is returned when a decision is requested before the model is installed.
var ErrNotReady = errors.New("the decision model is not installed; open the companion panel to install it")

type file struct {
	name string
	size int64
	sha  string
}

var modelFiles = []file{
	{"onnx/decision.data", 1522925568, "631417aa49dacb602b7e099062cd44c89e3747533a4094a61ff5f1cb3ccb0daf"},
	{"onnx/vision.data", 343372800, "490b694cf9a8770793e1cccc3d698bfb076f4b177216f9036e82eb0735b09390"},
	{"onnx/projector.data", 33566720, "e52c40550fbce097580870a7ddcca3f6992d93942931039ea717878514dd7a47"},
	{"onnx/decision.onnx", 366403, "ef66e7aa7f288fd71578cb6f6b71a04d6f531ad7755536487e155f66f605baf9"},
	{"onnx/vision.onnx", 119183, "aa7cbbabc0ee925ab4dd44a6c85dd7bf92ef36da5f2ebcdf1066f918d1a2ea89"},
	{"onnx/projector.onnx", 5712, "5489cd6636df06e474866ea55a37b06ec30a9fffcc993c89caff76639cc07d1e"},
	{"tokenizer.json", 4733371, "1efc3a6609abf6b63b1f47188d139f3b59973a6a434dffe970a7261a51ed2711"},
	{"config.json", 1765, "68ddf4bd4f3e489455211f80836b36281115cb9206b8e8d47858d062bdb6508d"},
	{"tiling-ratios.json", 681, "256b18bf3b533c9aaa34099b049e688893282341fb078b772b3ecb7b59ab43b6"},
}

type ortAsset struct {
	archive string
	size    int64
	sha     string
	lib     string // library path inside the archive (after the top folder)
}

func ortForPlatform() (ortAsset, bool) {
	switch runtime.GOOS + "/" + runtime.GOARCH {
	case "linux/amd64":
		return ortAsset{"onnxruntime-linux-x64-" + ortVersion + ".tgz", 11408003, ortSHA["linux-x64"], "lib/libonnxruntime.so." + ortVersion}, true
	case "linux/arm64":
		return ortAsset{"onnxruntime-linux-aarch64-" + ortVersion + ".tgz", 10331104, ortSHA["linux-aarch64"], "lib/libonnxruntime.so." + ortVersion}, true
	case "darwin/arm64":
		return ortAsset{"onnxruntime-osx-arm64-" + ortVersion + ".tgz", 36549009, ortSHA["osx-arm64"], "lib/libonnxruntime." + ortVersion + ".dylib"}, true
	case "windows/amd64":
		return ortAsset{"onnxruntime-win-x64-" + ortVersion + ".zip", 82887349, ortSHA["win-x64"], "lib/onnxruntime.dll"}, true
	case "windows/arm64":
		return ortAsset{"onnxruntime-win-arm64-" + ortVersion + ".zip", 85806468, ortSHA["win-arm64"], "lib/onnxruntime.dll"}, true
	}
	return ortAsset{}, false
}

func modelDir() string { return filepath.Join(paths.Models(), ModelName) }
func ortDir() string   { return filepath.Join(paths.Runtimes(), "onnxruntime-"+ortVersion) }

func ortLibPath() string {
	a, ok := ortForPlatform()
	if !ok {
		return ""
	}
	return filepath.Join(ortDir(), filepath.FromSlash(a.lib))
}

type ortMarker struct {
	ArchiveSHA  string `json:"archive_sha"`
	LibrarySize int64  `json:"library_size"`
	LibrarySHA  string `json:"library_sha"`
}

func ortMarkerPath() string { return filepath.Join(ortDir(), ".installed") }

type Status struct {
	Model         string             `json:"model"`
	Phase         string             `json:"phase"`
	Step          string             `json:"step,omitempty"`
	Progress      *download.Progress `json:"progress,omitempty"`
	Error         string             `json:"error,omitempty"`
	Installed     bool               `json:"installed"`
	Supported     bool               `json:"supported"`
	Enabled       bool               `json:"enabled"`
	Runtime       string             `json:"runtime,omitempty"`
	DownloadBytes int64              `json:"download_bytes"`
	DiskBytes     int64              `json:"disk_bytes"`
	Endpoint      string             `json:"endpoint"`
	Requests      int64              `json:"requests"`
	LastLatencyMs float64            `json:"last_latency_ms,omitempty"`
	License       string             `json:"license"`
}

type Manager struct {
	mu          sync.Mutex
	store       *settings.Store
	onChange    func()
	phase       string
	step        string
	progress    *download.Progress
	err         string
	cancel      context.CancelFunc
	installDone chan struct{}
	removing    bool
	closing     bool
	eng         *engine
	evalMu      sync.Mutex
	verified    download.Verifier
	requests    int64
	latency     float64
}

func NewManager(store *settings.Store, onChange func()) *Manager {
	return &Manager{store: store, onChange: onChange, phase: PhaseIdle}
}

func (m *Manager) changed() {
	if m.onChange != nil {
		go m.onChange()
	}
}

func okFile(path string, size int64) bool {
	st, err := os.Stat(path)
	return err == nil && st.Size() == size
}

func (m *Manager) validFile(path string, size int64, sha string) bool {
	return m.verified.Valid(path, size, sha)
}

// The library may exist after an interrupted extraction. Only a marker
// written after the full extraction and verified against the library makes
// this runtime usable.
func (m *Manager) ortReady(asset ortAsset) bool {
	data, err := os.ReadFile(ortMarkerPath())
	if err != nil {
		return false
	}
	var marker ortMarker
	if json.Unmarshal(data, &marker) != nil || marker.ArchiveSHA != asset.sha || marker.LibrarySize <= 0 || marker.LibrarySHA == "" {
		return false
	}
	return m.validFile(ortLibPath(), marker.LibrarySize, marker.LibrarySHA)
}

func (m *Manager) markORT(asset ortAsset) error {
	path := ortLibPath()
	st, err := os.Stat(path)
	if err != nil || !st.Mode().IsRegular() || st.Size() == 0 {
		return errors.New("ONNX Runtime library missing from the archive")
	}
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	h := sha256.New()
	_, err = io.Copy(h, f)
	_ = f.Close()
	if err != nil {
		return err
	}
	marker := ortMarker{ArchiveSHA: asset.sha, LibrarySize: st.Size(), LibrarySHA: hex.EncodeToString(h.Sum(nil))}
	data, err := json.Marshal(marker)
	if err != nil {
		return err
	}
	tmp := ortMarkerPath() + ".tmp"
	if err := os.WriteFile(tmp, data, 0o600); err != nil {
		return err
	}
	return os.Rename(tmp, ortMarkerPath())
}

// Installed reports whether the model files and runtime are present.
func (m *Manager) Installed() bool {
	asset, ok := ortForPlatform()
	if !ok || !m.ortReady(asset) {
		return false
	}
	for _, f := range modelFiles {
		if !m.validFile(filepath.Join(modelDir(), filepath.FromSlash(f.name)), f.size, f.sha) {
			return false
		}
	}
	return true
}

func (m *Manager) Status() Status {
	_, supported := ortForPlatform()
	st := Status{Model: ModelName, Supported: supported, Installed: m.Installed(), Enabled: m.store.Get().Decision.Enabled,
		License: "LFM Open License v1.0"}
	for _, f := range modelFiles {
		p := filepath.Join(modelDir(), filepath.FromSlash(f.name))
		if m.validFile(p, f.size, f.sha) {
			st.DiskBytes += f.size
			continue
		}
		rem := f.size
		if s, err := os.Stat(p + ".part"); err == nil && s.Size() < f.size {
			rem -= s.Size()
		}
		st.DownloadBytes += rem
	}
	if a, ok := ortForPlatform(); ok {
		if !m.ortReady(a) {
			st.DownloadBytes += a.size
		}
	}
	m.mu.Lock()
	st.Phase, st.Step, st.Error = m.phase, m.step, m.err
	if m.progress != nil {
		p := *m.progress
		st.Progress = &p
	}
	if m.eng != nil {
		st.Runtime = "ONNX Runtime " + m.eng.ort.version + " on the CPU"
	}
	st.Requests, st.LastLatencyMs = m.requests, m.latency
	m.mu.Unlock()
	return st
}

func (m *Manager) set(phase, step string, err error) {
	m.mu.Lock()
	m.phase, m.step = phase, step
	m.err = ""
	if err != nil {
		m.err = err.Error()
	}
	if phase != PhaseDownloading {
		m.progress = nil
	}
	m.mu.Unlock()
	m.changed()
}

// Install downloads the model and runtime, then loads them.
func (m *Manager) Install() error {
	asset, ok := ortForPlatform()
	if !ok {
		return errors.New("ONNX Runtime " + ortVersion + " has no build for this platform")
	}
	m.mu.Lock()
	if m.closing {
		m.mu.Unlock()
		return errors.New("the companion is shutting down")
	}
	if m.removing || m.phase == PhaseDownloading || m.phase == PhaseInstalling || m.phase == PhaseLoading {
		m.mu.Unlock()
		return errors.New("already installing")
	}
	ctx, cancel := context.WithCancel(context.Background())
	m.cancel = cancel
	done := make(chan struct{})
	m.installDone = done
	m.phase, m.step, m.err, m.progress = PhaseDownloading, "Preparing the decision model", "", nil
	m.mu.Unlock()
	m.changed()
	go func() {
		defer cancel()
		defer close(done)
		if err := m.install(ctx, asset); err != nil {
			if errors.Is(err, context.Canceled) {
				m.set(PhaseIdle, "", nil)
				return
			}
			logx.Printf("decision model install: %v", err)
			m.set(PhaseError, "", err)
			return
		}
		if err := ctx.Err(); err != nil {
			m.set(PhaseIdle, "", nil)
			return
		}
		if _, err := m.store.Update(func(s *settings.Settings) { s.Decision.Enabled = true }); err != nil {
			m.set(PhaseError, "", fmt.Errorf("save decision-model preference: %w", err))
			return
		}
		if err := m.Load(); err != nil {
			logx.Printf("decision model load: %v", err)
		}
	}()
	return nil
}

func (m *Manager) Cancel() {
	m.mu.Lock()
	c := m.cancel
	m.mu.Unlock()
	if c != nil {
		c()
	}
}

// BeginShutdown cancels installation and waits for its worker to release
// downloads and any newly loaded sessions before Unload runs.
func (m *Manager) BeginShutdown() {
	m.mu.Lock()
	m.closing = true
	cancel, done := m.cancel, m.installDone
	m.mu.Unlock()
	if cancel != nil {
		cancel()
	}
	if done != nil {
		<-done
	}
}

func (m *Manager) install(ctx context.Context, asset ortAsset) error {
	type item struct {
		label, url, dest string
		size             int64
		sha              string
	}
	var items []item
	for _, f := range modelFiles {
		items = append(items, item{"d1 decision model", fmt.Sprintf("https://huggingface.co/%s/resolve/%s/%s", repo, revision, f.name),
			filepath.Join(modelDir(), filepath.FromSlash(f.name)), f.size, f.sha})
	}
	archive := filepath.Join(paths.Runtimes(), "_downloads", asset.archive)
	needORT := !m.ortReady(asset)
	if needORT {
		items = append(items, item{"ONNX Runtime " + ortVersion, "https://github.com/microsoft/onnxruntime/releases/download/v" + ortVersion + "/" + asset.archive,
			archive, asset.size, asset.sha})
	}
	var total, done int64
	for _, it := range items {
		if !m.validFile(it.dest, it.size, it.sha) {
			total += it.size
		}
	}
	for _, it := range items {
		if m.validFile(it.dest, it.size, it.sha) {
			continue
		}
		var before int64
		if s, err := os.Stat(it.dest + ".part"); err == nil {
			before = s.Size()
		}
		base := done
		m.mu.Lock()
		m.phase, m.step = PhaseDownloading, it.label
		m.progress = &download.Progress{Done: base + before, Total: total}
		m.mu.Unlock()
		m.changed()
		err := download.File(ctx, it.url, it.dest, it.size, it.sha, func(p download.Progress) {
			m.mu.Lock()
			m.progress = &download.Progress{Done: base + p.Done, Total: total, BytesPerSec: p.BytesPerSec}
			m.mu.Unlock()
			m.changed()
		})
		if err != nil {
			return fmt.Errorf("%s: %w", it.label, err)
		}
		done = base + it.size
	}
	if needORT {
		m.set(PhaseInstalling, "Unpacking ONNX Runtime", nil)
		if err := os.RemoveAll(ortDir()); err != nil {
			return fmt.Errorf("replace ONNX Runtime: %w", err)
		}
		keep := func(rel string) bool {
			// Only the runtime libraries: skip headers, symbols (a 430 MB .pdb) and test data.
			if !strings.HasPrefix(rel, "lib/") || strings.Contains(rel, ".dSYM") {
				return false
			}
			return strings.HasSuffix(rel, ".dll") || strings.HasSuffix(rel, ".dylib") || strings.Contains(rel, ".so")
		}
		if err := download.ExtractFiltered(archive, ortDir(), download.CommonPrefixDepth(archive), keep); err != nil {
			return fmt.Errorf("unpack ONNX Runtime: %w", err)
		}
		if runtime.GOOS == "darwin" {
			trustMac(ortDir())
		}
		if err := m.markORT(asset); err != nil {
			return err
		}
		_ = os.Remove(archive)
	}
	return nil
}

// Load creates the ONNX Runtime sessions (about 2 GB of memory).
func (m *Manager) Load() error {
	m.evalMu.Lock()
	defer m.evalMu.Unlock()
	return m.load()
}

// load requires evalMu so sessions cannot be replaced or released during use.
func (m *Manager) load() error {
	m.mu.Lock()
	closing := m.closing
	m.mu.Unlock()
	if closing {
		return errors.New("the companion is shutting down")
	}
	if !m.Installed() {
		return ErrNotReady
	}
	m.mu.Lock()
	if m.eng != nil {
		m.mu.Unlock()
		return nil
	}
	m.mu.Unlock()
	m.set(PhaseLoading, "Loading graphs", nil)
	threads := m.store.Get().Decision.Threads
	eng, err := loadEngine(modelDir(), ortLibPath(), threads)
	if err != nil {
		m.set(PhaseError, "", err)
		return err
	}
	m.mu.Lock()
	m.eng = eng
	m.mu.Unlock()
	m.set(PhaseReady, "", nil)
	logx.Printf("decision model ready (ONNX Runtime %s)", eng.ort.version)
	return nil
}

// Unload releases the sessions (the runtime library stays mapped).
func (m *Manager) Unload() {
	m.evalMu.Lock()
	defer m.evalMu.Unlock()
	m.unload()
}

// unload requires evalMu.
func (m *Manager) unload() {
	m.mu.Lock()
	eng := m.eng
	m.eng = nil
	if m.phase == PhaseReady {
		m.phase = PhaseIdle
	}
	m.mu.Unlock()
	if eng != nil {
		eng.close()
	}
	m.changed()
}

// Remove deletes the model files.
func (m *Manager) Remove() error {
	m.mu.Lock()
	if m.removing {
		m.mu.Unlock()
		return errors.New("already removing the decision model")
	}
	m.removing = true
	cancel := m.cancel
	done := m.installDone
	m.mu.Unlock()
	defer func() {
		m.mu.Lock()
		m.removing = false
		m.mu.Unlock()
	}()
	if cancel != nil {
		cancel()
	}
	if done != nil {
		<-done
	}
	if _, err := m.store.Update(func(s *settings.Settings) { s.Decision.Enabled = false }); err != nil {
		return err
	}
	m.evalMu.Lock()
	defer m.evalMu.Unlock()
	m.unload()
	if err := os.RemoveAll(modelDir()); err != nil {
		return err
	}
	m.set(PhaseIdle, "", nil)
	return nil
}

// engine requires evalMu to keep the returned sessions alive for the caller.
func (m *Manager) engine() (*engine, error) {
	m.mu.Lock()
	eng := m.eng
	m.mu.Unlock()
	if eng != nil {
		return eng, nil
	}
	if !m.Installed() {
		return nil, ErrNotReady
	}
	if err := m.load(); err != nil {
		return nil, err
	}
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.eng == nil {
		return nil, ErrNotReady
	}
	return m.eng, nil
}

// EvaluateJSON answers one {state, images, questions} request.
func (m *Manager) EvaluateJSON(ctx context.Context, body []byte) (any, error) {
	req, err := parseRequest(body)
	if err != nil {
		return nil, err
	}
	m.evalMu.Lock()
	defer m.evalMu.Unlock()
	eng, err := m.engine()
	if err != nil {
		return nil, err
	}
	start := time.Now()
	res, err := eng.evaluate(ctx, req, ModelName)
	if err != nil {
		return nil, err
	}
	m.mu.Lock()
	m.requests++
	m.latency = float64(time.Since(start).Microseconds()) / 1000
	m.mu.Unlock()
	m.changed()
	return res, nil
}

// SelfTest runs a small text-only decision.
func (m *Manager) SelfTest(ctx context.Context) (any, error) {
	body, _ := json.Marshal(map[string]any{
		"state": "Checkout complete. Order #1042 is confirmed and a receipt was sent to your email.",
		"questions": json.RawMessage(`{"completion":{"type":"choice","instructions":"Judge only the visible evidence: did the shopping task finish?",` +
			`"criteria":{"completed":"An explicit confirmation shows the order was placed.","not_completed":"The page shows an error or an unfinished checkout.","unknown":"The evidence does not establish the outcome."}}}`),
	})
	return m.EvaluateJSON(ctx, body)
}

func trustMac(dir string) {
	if runtime.GOOS != "darwin" {
		return
	}
	_ = filepath.Walk(dir, func(p string, info os.FileInfo, err error) error {
		if err == nil && !info.IsDir() && strings.HasSuffix(p, ".dylib") {
			_ = execQuiet("/usr/bin/xattr", "-c", p)
			_ = execQuiet("/usr/bin/codesign", "-s", "-", "--force", "--timestamp=none", p)
		}
		return nil
	})
}