Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
8.5 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//go:build !cgo || !switchyard_native

package switchyard

import (
	"bytes"
	"context"
	cryptorand "crypto/rand"
	_ "embed"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"sync"

	"github.com/tetratelabs/wazero"
	"github.com/tetratelabs/wazero/api"
	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
)

//go:embed internal/wasm/switchyard.wasm
var switchyardWasm []byte

const wasmABIVersion = 1

type wasmRuntime struct {
	runtime  wazero.Runtime
	compiled wazero.CompiledModule
	close    sync.Once
}

var wasmCompilationCache = wazero.NewCompilationCache()

type wasmTrapError struct {
	function string
	err      error
	stderr   string
}

func (err *wasmTrapError) Error() string {
	if err.stderr != "" {
		return fmt.Sprintf("Switchyard wasm %s trapped: %v: %s", err.function, err.err, err.stderr)
	}
	return fmt.Sprintf("Switchyard wasm %s trapped: %v", err.function, err.err)
}
func (err *wasmTrapError) Unwrap() error { return err.err }

type wasmGuestError struct{ message string }

func (err *wasmGuestError) Error() string { return err.message }

func isWasmTrap(err error) bool {
	var target *wasmTrapError
	return errors.As(err, &target)
}

func newWasmRuntime() (*wasmRuntime, error) {
	config := wazero.NewRuntimeConfigCompiler().
		WithCompilationCache(wasmCompilationCache).
		WithMemoryLimitPages(16384).
		WithCloseOnContextDone(true)
	runtime := wazero.NewRuntimeWithConfig(context.Background(), config)
	if _, err := wasi_snapshot_preview1.Instantiate(context.Background(), runtime); err != nil {
		_ = runtime.Close(context.Background())
		return nil, fmt.Errorf("instantiate WASI: %w", err)
	}
	compiled, err := runtime.CompileModule(context.Background(), switchyardWasm)
	if err != nil {
		_ = runtime.Close(context.Background())
		return nil, fmt.Errorf("compile Switchyard wasm: %w", err)
	}
	return &wasmRuntime{runtime: runtime, compiled: compiled}, nil
}

// guestStderr keeps the guest's latest stderr output, such as a Rust panic message.
type guestStderr struct {
	mu  sync.Mutex
	buf bytes.Buffer
}

const guestStderrLimit = 4096

func (stderr *guestStderr) Write(data []byte) (int, error) {
	stderr.mu.Lock()
	defer stderr.mu.Unlock()
	stderr.buf.Write(data)
	if extra := stderr.buf.Len() - guestStderrLimit; extra > 0 {
		stderr.buf.Next(extra)
	}
	return len(data), nil
}

func (stderr *guestStderr) take() string {
	stderr.mu.Lock()
	defer stderr.mu.Unlock()
	text := strings.TrimSpace(stderr.buf.String())
	stderr.buf.Reset()
	return text
}

func (runtime *wasmRuntime) newModule(ctx context.Context) (*guestModule, error) {
	stderr := &guestStderr{}
	config := wazero.NewModuleConfig().
		WithStderr(stderr).
		WithStartFunctions().
		WithSysWalltime().
		WithSysNanotime().
		WithRandSource(cryptorand.Reader).
		WithName("")
	module, err := runtime.runtime.InstantiateModule(ctx, runtime.compiled, config)
	if err != nil {
		return nil, err
	}
	guest := &guestModule{module: module, stderr: stderr}
	if _, err := guest.call(ctx, "_initialize"); err != nil {
		_ = module.Close(context.Background())
		return nil, fmt.Errorf("initialize Switchyard wasm: %w", err)
	}
	version, err := guest.call(ctx, "sy_abi_version")
	if err != nil {
		_ = module.Close(context.Background())
		return nil, fmt.Errorf("read Switchyard wasm ABI: %w", err)
	}
	if len(version) != 1 || uint32(version[0]) != wasmABIVersion {
		_ = module.Close(context.Background())
		return nil, fmt.Errorf("Switchyard wasm ABI version mismatch: module=%v, binding=%d", version, wasmABIVersion)
	}
	return guest, nil
}

func (runtime *wasmRuntime) closeAll() {
	runtime.close.Do(func() {
		_ = runtime.runtime.Close(context.Background())
	})
}

type guestModule struct {
	module api.Module
	stderr *guestStderr
}

func (guest *guestModule) call(ctx context.Context, name string, args ...uint64) ([]uint64, error) {
	function := guest.module.ExportedFunction(name)
	if function == nil {
		return nil, fmt.Errorf("Switchyard wasm export %q is missing", name)
	}
	values, err := function.Call(ctx, args...)
	if err != nil {
		return nil, &wasmTrapError{function: name, err: err, stderr: guest.stderr.take()}
	}
	return values, nil
}

func (guest *guestModule) alloc(ctx context.Context, data []byte) (uint32, error) {
	if uint64(len(data)) > uint64(^uint32(0)) {
		return 0, errors.New("Switchyard wasm buffer exceeds wasm32 memory")
	}
	values, err := guest.call(ctx, "sy_alloc", uint64(uint32(len(data))))
	if err != nil {
		return 0, err
	}
	if len(values) != 1 || uint32(values[0]) == 0 {
		return 0, errors.New("Switchyard wasm allocation failed")
	}
	ptr := uint32(values[0])
	if !guest.module.Memory().Write(ptr, data) {
		_, _ = guest.call(context.Background(), "sy_free", uint64(ptr), uint64(uint32(len(data))))
		return 0, errors.New("Switchyard wasm allocation is outside guest memory")
	}
	return ptr, nil
}

func (guest *guestModule) free(ctx context.Context, ptr uint32, length uint32) error {
	_, err := guest.call(ctx, "sy_free", uint64(ptr), uint64(length))
	return err
}

func (guest *guestModule) cstring(ctx context.Context, data []byte) (uint32, uint32, error) {
	data = append(bytes.Clone(data), 0)
	ptr, err := guest.alloc(ctx, data)
	return ptr, uint32(len(data)), err
}

func (guest *guestModule) readCString(ptr uint32) ([]byte, error) {
	if ptr == 0 {
		return nil, errors.New("Switchyard wasm returned a null string")
	}
	memory := guest.module.Memory()
	size := memory.Size()
	if size == 0 || ptr >= size {
		return nil, errors.New("Switchyard wasm string pointer is outside guest memory")
	}
	const chunkSize = uint32(16 * 1024)
	output := make([]byte, 0, chunkSize)
	for offset := ptr; offset < size; {
		count := min(chunkSize, size-offset)
		chunk, ok := memory.Read(offset, count)
		if !ok {
			return nil, errors.New("Switchyard wasm string is outside guest memory")
		}
		if end := bytes.IndexByte(chunk, 0); end >= 0 {
			output = append(output, chunk[:end]...)
			return output, nil
		}
		output = append(output, chunk...)
		offset += count
	}
	return nil, errors.New("Switchyard wasm returned an unterminated string")
}

func (guest *guestModule) guestError(ptr uint32) error {
	if ptr == 0 {
		return nil
	}
	message, err := guest.readCString(ptr)
	_, freeErr := guest.call(context.Background(), "sy_string_free", uint64(ptr))
	if err != nil {
		return err
	}
	if freeErr != nil {
		return freeErr
	}
	return &wasmGuestError{message: string(message)}
}

func (guest *guestModule) callJSON(ctx context.Context, export string, input []byte) ([]byte, error) {
	inputPtr, inputLen, err := guest.cstring(ctx, input)
	if err != nil {
		return nil, err
	}
	defer guest.free(context.Background(), inputPtr, inputLen)
	outPtr, err := guest.alloc(ctx, make([]byte, 4))
	if err != nil {
		return nil, err
	}
	defer guest.free(context.Background(), outPtr, 4)
	values, err := guest.call(ctx, export, uint64(inputPtr), uint64(outPtr))
	if err != nil {
		return nil, err
	}
	if len(values) != 1 {
		return nil, fmt.Errorf("Switchyard wasm %s returned %d values", export, len(values))
	}
	if err := guest.guestError(uint32(values[0])); err != nil {
		return nil, err
	}
	outputPtr, ok := guest.module.Memory().ReadUint32Le(outPtr)
	if !ok || outputPtr == 0 {
		return nil, errors.New("Switchyard wasm returned no JSON output")
	}
	output, err := guest.readCString(outputPtr)
	if err != nil {
		return nil, err
	}
	if _, err := guest.call(context.Background(), "sy_string_free", uint64(outputPtr)); err != nil {
		return nil, err
	}
	return output, nil
}

type wasmTranslator struct {
	runtime *wasmRuntime
	guest   *guestModule
	err     error
	mu      sync.Mutex
}

var defaultWasmTranslator struct {
	once  sync.Once
	value *wasmTranslator
}

func wasmTranslate(export string, input []byte) (json.RawMessage, error) {
	defaultWasmTranslator.once.Do(func() {
		runtime, err := newWasmRuntime()
		defaultWasmTranslator.value = &wasmTranslator{runtime: runtime, err: err}
	})
	translator := defaultWasmTranslator.value
	if translator.err != nil {
		return nil, translator.err
	}
	translator.mu.Lock()
	defer translator.mu.Unlock()
	if translator.guest == nil {
		guest, err := translator.runtime.newModule(context.Background())
		if err != nil {
			return nil, err
		}
		translator.guest = guest
	}
	output, err := translator.guest.callJSON(context.Background(), export, input)
	if err != nil && (isWasmTrap(err) || translator.guest.module.IsClosed()) {
		_ = translator.guest.module.Close(context.Background())
		translator.guest = nil
	}
	return json.RawMessage(output), err
}