A high-performance inference engine for AI models
diff --git a/install/inprocess.py b/install/inprocess.py
new file mode 100644
index 0000000..bb863e7
--- /dev/null
+++ b/install/inprocess.py
@@ -0,0 +1,103 @@
+"""The engine's commands inside this process, where none can be started (iOS),
+with the subset of subprocess.run and subprocess.Popen that Splash uses."""
+
+import ctypes
+import os
+import subprocess
+import threading
+from functools import cache
+
+from . import paths
+
+
+@cache
+def _engine():
+ engine = ctypes.CDLL(str(paths.BINARY))
+ engine.benchmarks_splash_main.argtypes = [
+ ctypes.c_int,
+ ctypes.POINTER(ctypes.c_char_p),
+ ctypes.c_int,
+ ctypes.c_int,
+ ]
+ engine.benchmarks_splash_terminate.argtypes = []
+ return engine
+
+
+def _main(args, input, output):
+ argv = (ctypes.c_char_p * (len(args) + 1))(*(str(arg).encode() for arg in args), None)
+ returncode = _engine().benchmarks_splash_main(len(args), argv, input, output)
+ ctypes.CDLL(None).fflush(None)
+ return returncode
+
+
+def _drain(descriptor, chunks):
+ with os.fdopen(descriptor, "rb") as stream:
+ chunks.append(stream.read())
+
+
+def run(args, *, check=False, capture_output=False, text=False):
+ """Run a check that writes to stdout and stderr, as subprocess.run does."""
+ if not capture_output:
+ returncode = _main(args, 0, 1)
+ stdout = stderr = None
+ else:
+ saved = [os.dup(1), os.dup(2)]
+ pipes = [os.pipe(), os.pipe()]
+ outputs = [[], []]
+ readers = [threading.Thread(target=_drain, args=(pipe[0], chunks)) for pipe, chunks in zip(pipes, outputs)]
+ for reader in readers:
+ reader.start()
+ try:
+ for descriptor, pipe in zip((1, 2), pipes):
+ os.dup2(pipe[1], descriptor)
+ os.close(pipe[1])
+ returncode = _main(args, 0, 1)
+ finally:
+ for descriptor, copy in zip((1, 2), saved):
+ os.dup2(copy, descriptor)
+ os.close(copy)
+ for reader in readers:
+ reader.join()
+ stdout, stderr = (b"".join(chunks) for chunks in outputs)
+ if text:
+ stdout, stderr = stdout.decode(), stderr.decode()
+ if check and returncode:
+ raise subprocess.CalledProcessError(returncode, args, stdout, stderr)
+ return subprocess.CompletedProcess(args, returncode, stdout, stderr)
+
+
+class Popen:
+ """The native server on a thread, with pipes for its frames and the
+ shared stderr, as subprocess.Popen(stdin=PIPE, stdout=PIPE, bufsize=0)."""
+
+ def __init__(self, args, **_):
+ self.args = args
+ self.pid = os.getpid()
+ self.returncode = None
+ input, writer = os.pipe()
+ reader, output = os.pipe()
+ self.stdin = os.fdopen(writer, "wb", buffering=0)
+ self.stdout = os.fdopen(reader, "rb", buffering=0)
+ self._thread = threading.Thread(target=self._serve, args=(input, output), daemon=True)
+ self._thread.start()
+
+ def _serve(self, input, output):
+ try:
+ self.returncode = _main(self.args, input, output)
+ finally:
+ os.close(input)
+ os.close(output)
+
+ def poll(self):
+ return None if self._thread.is_alive() else self.returncode
+
+ def wait(self, timeout=None):
+ self._thread.join(timeout)
+ if self._thread.is_alive():
+ raise subprocess.TimeoutExpired(self.args, timeout)
+ return self.returncode
+
+ def terminate(self):
+ _engine().benchmarks_splash_terminate()
+
+ kill = terminate
diff --git a/install/models.py b/install/models.py
index cb6033c..593ee13 100644
--- a/install/models.py
+++ b/install/models.py
@@ -41,9 +41,11 @@ if __name__ == "__main__" and not __package__:
# the standard library alone: the launcher imports this before .venv exists.
from server import serve_options
-from . import paths
+from . import inprocess, paths
MODELS = paths.MODELS
+# The engine's checks run in this process where none can be started.
+processes = inprocess if paths.SANDBOXED else subprocess
# The bound on one JSON metadata file the installer reads, a larger one being
# refused unread: configs are kilobytes, and a shard index names every tensor,
# 14 MB for a checkpoint that stores each routed expert's tensors of its own
@@ -89,7 +91,7 @@ def run_engine(arguments, check):
engine = paths.BINARY
replace = "reinstall Splash" if paths.PACKAGED else "rebuild it with make"
try:
- result = subprocess.run(
+ result = processes.run(
[str(engine), *arguments], capture_output=True, text=True
)
except OSError as error:
diff --git a/install/paths.py b/install/paths.py
index 4509cc6..5c2eff8 100644
--- a/install/paths.py
+++ b/install/paths.py
@@ -1,11 +1,18 @@
"""Immutable program files and writable per-user data, for source or release."""
+import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
PACKAGED = (ROOT / "release.json").is_file()
-DATA = Path.home() / "Library/Application Support/Splash" if PACKAGED else ROOT
-MODELS = DATA / "models" if PACKAGED else ROOT / "install/models"
-RUNTIME = DATA / "runtime" if PACKAGED else ROOT / "build/runtime"
+SANDBOXED = sys.platform == "ios"
+PER_USER = PACKAGED or SANDBOXED
+DATA = Path.home() / "Library/Application Support/Splash" if PER_USER else ROOT
+MODELS = DATA / "models" if PER_USER else ROOT / "install/models"
+RUNTIME = DATA / "runtime" if PER_USER else ROOT / "build/runtime"
PYTHON = ROOT / ("python/bin/python3" if PACKAGED else ".venv/bin/python")
-BINARY = ROOT / ("engine/splash" if PACKAGED else "build/splash")
+BINARY = (
+ Path(sys.executable).parent / "Frameworks/BenchmarksSplash.framework/BenchmarksSplash"
+ if SANDBOXED
+ else ROOT / ("engine/splash" if PACKAGED else "build/splash")
+)
diff --git a/runtime/ane/Program.mm b/runtime/ane/Program.mm
index 3840533..90e30a7 100644
--- a/runtime/ane/Program.mm
+++ b/runtime/ane/Program.mm
@@ -6,7 +6,7 @@
#endif
#import <Foundation/Foundation.h>
-#import <IOSurface/IOSurface.h>
+#import <IOSurface/IOSurfaceRef.h>
#include <dispatch/dispatch.h>
#include <dlfcn.h>
diff --git a/runtime/main.mm b/runtime/main.mm
index 3e157b1..67363f4 100644
--- a/runtime/main.mm
+++ b/runtime/main.mm
@@ -7,9 +7,12 @@
#include "engine/Status.hpp"
#include "model/ModelDescriptor.hpp"
+#include <TargetConditionals.h>
+#if TARGET_OS_OSX
#include <IOKit/pwr_mgt/IOPMLib.h>
+#endif
+#include <dlfcn.h>
#include <dispatch/dispatch.h>
-#include <mach-o/dyld.h>
#include <mach/mach_error.h>
#include <sysexits.h>
#include <unistd.h>
@@ -22,7 +25,6 @@
#include <cstdint>
#include <cstdlib>
#include <filesystem>
-#include <limits.h>
#include <memory>
#include <optional>
#include <stdexcept>
@@ -31,7 +33,6 @@
#include <system_error>
#include <thread>
#include <utility>
-#include <vector>
#ifndef SPLASH_BUILD_ID
#error "production build requires the generated BuildIdentity.hpp"
@@ -108,6 +109,7 @@ private:
// does: the display may still sleep, and the lid, the Sleep command or a low
// battery still sleep the Mac; requests continue when it wakes. Without the
// assertion the engine serves on and says so once.
+#if TARGET_OS_OSX
class IdleSleepAssertion final {
public:
IdleSleepAssertion() = default;
@@ -140,6 +142,7 @@ private:
IOPMAssertionID assertion_ = kIOPMNullAssertionID;
bool reported_ = false;
};
+#endif
void printUsage(std::string_view executable) {
const std::string command(executable);
@@ -159,16 +162,11 @@ void printUsage(std::string_view executable) {
}
std::filesystem::path executablePath() {
- uint32_t size = PATH_MAX;
- std::vector<char> buffer(size);
- if (_NSGetExecutablePath(buffer.data(), &size) != 0) {
- buffer.resize(size);
- if (_NSGetExecutablePath(buffer.data(), &size) != 0) {
- throw std::runtime_error("could not resolve executable path");
- }
- }
+ Dl_info info{};
+ if (!dladdr(reinterpret_cast<const void *>(&executablePath), &info) || !info.dli_fname)
+ throw std::runtime_error("could not resolve executable path");
std::error_code error;
- std::filesystem::path path = std::filesystem::canonical(buffer.data(), error);
+ std::filesystem::path path = std::filesystem::canonical(info.dli_fname, error);
if (error) {
throw std::runtime_error("could not canonicalize executable path: " +
error.message());
@@ -242,20 +240,27 @@ public:
action.sa_handler = requestShutdownFromSignal;
sigemptyset(&action.sa_mask);
action.sa_flags = 0;
- for (const int number : {SIGTERM, SIGINT, SIGHUP}) {
- struct sigaction inherited {};
- if (number == SIGHUP && sigaction(number, nullptr, &inherited) == 0 &&
- inherited.sa_handler == SIG_IGN)
+ struct sigaction ignore {};
+ ignore.sa_handler = SIG_IGN;
+ for (std::size_t index = 0; index < std::size(kSignals); ++index) {
+ const int number = kSignals[index];
+ sigaction(number, nullptr, &inherited_[index]);
+ if (number == SIGHUP && inherited_[index].sa_handler == SIG_IGN)
continue;
- sigaction(number, &action, nullptr);
+ sigaction(number, number == SIGPIPE ? &ignore : &action, nullptr);
}
- std::signal(SIGPIPE, SIG_IGN);
}
ShutdownSignals(const ShutdownSignals &) = delete;
ShutdownSignals &operator=(const ShutdownSignals &) = delete;
~ShutdownSignals() {
+ for (std::size_t index = 0; index < std::size(kSignals); ++index)
+ sigaction(kSignals[index], &inherited_[index], nullptr);
gShutdownTransport.store(nullptr, std::memory_order_release);
}
+
+private:
+ static constexpr int kSignals[] = {SIGTERM, SIGINT, SIGHUP, SIGPIPE};
+ struct sigaction inherited_[std::size(kSignals)] {};
};
// The time a clean stop gives the newest restore points of a persistent
@@ -302,13 +307,15 @@ void reportStartFailure(engine::FdTransport &transport,
}
}
-int runNative(const engine::NativeArguments &arguments) {
+int runNative(const engine::NativeArguments &arguments, int input, int output) {
if (const std::filesystem::path caches = splashCaches(); !caches.empty())
reportLegacyWeightCache(caches / "weights");
- engine::FdTransport transport(STDIN_FILENO, STDOUT_FILENO);
+ engine::FdTransport transport(input, output);
ShutdownSignals signals(transport);
MemoryPressureMonitor pressureMonitor(transport.controlNotifier());
+#if TARGET_OS_OSX
IdleSleepAssertion idleSleep;
+#endif
engine::RuntimeMetrics metrics;
engine::RuntimeBootstrap *published = nullptr;
auto statusProvider = [&] {
@@ -327,8 +334,10 @@ int runNative(const engine::NativeArguments &arguments) {
config.resources.memoryPressure = [&] { return pressureMonitor.pressure(); };
config.resources.cancelled = [&] { return transport.shutdownRequested(); };
config.nativeLoop.metrics = &metrics;
+#if TARGET_OS_OSX
if (arguments.preventIdleSleep)
config.nativeLoop.holdingRequests = [&](bool held) { idleSleep.hold(held); };
+#endif
try {
bootstrap = engine::RuntimeBootstrap::start(
std::move(config), transport.outputSink(), statusProvider);
@@ -427,7 +436,9 @@ int checkModel(int argc, char **argv) {
} // namespace
} // namespace splash
-int main(int argc, char **argv) {
+extern "C" void benchmarks_splash_terminate() { splash::requestShutdownFromSignal(SIGTERM); }
+
+extern "C" int benchmarks_splash_main(int argc, char **argv, int input, int output) {
@autoreleasepool {
try {
if (argc == 2 && std::string_view(argv[1]) == "device-check")
@@ -436,7 +447,7 @@ int main(int argc, char **argv) {
return splash::checkModel(argc, argv);
const splash::engine::NativeArguments arguments =
splash::engine::parseNativeArguments(argc, argv);
- return splash::runNative(arguments);
+ return splash::runNative(arguments, input, output);
} catch (const splash::engine::UsageError &error) {
splash::writeStderrLine(std::string("error: ") + error.what());
splash::printUsage(argc > 0 ? argv[0] : "splash");
@@ -452,3 +463,7 @@ int main(int argc, char **argv) {
}
}
}
+
+int main(int argc, char **argv) {
+ return benchmarks_splash_main(argc, argv, STDIN_FILENO, STDOUT_FILENO);
+}