Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

strata-vision: HIP build (strata-vision-hip) for AMD GPUs; the reader refuses a GPU that is not in its build; RUNPATH $ORIGIN

Niko1221fb14c354b414
diff --git a/tools/vision/CMakeLists.txt b/tools/vision/CMakeLists.txt
index ed42e6254..5b09e2d6f 100644
--- a/tools/vision/CMakeLists.txt
+++ b/tools/vision/CMakeLists.txt
@@ -3,6 +3,7 @@
#
# cmake -S tools/vision -B build-vision -G Ninja -DCMAKE_BUILD_TYPE=Release -DLLAMA_DIR=third_party/llama.cpp
# [-DSTRATA_VISION_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native] [-DSTRATA_PORTABLE=OFF]
+# [-DSTRATA_VISION_HIP=ON -DSTRATA_VISION_HIP_ARCHS="gfx1200;gfx1201" + the HIP compiler the engine build uses]
# cmake --build build-vision --target strata-vision
#
# Portable (AVX2 baseline) is the default, so an encoder built without extra flags runs on any AVX2 PC: the 0.1.32
@@ -14,6 +15,14 @@ project(strata_vision C CXX)
set(LLAMA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../third_party/llama.cpp" CACHE PATH "llama.cpp checkout")
option(STRATA_VISION_CUDA "run the vision encoder on the GPU (builds ggml-cuda)" OFF)
+option(STRATA_VISION_HIP "run the vision encoder on an AMD GPU (builds ggml-hip); the program is named strata-vision-hip" OFF)
+set(STRATA_VISION_HIP_ARCHS "" CACHE STRING "the gfx architectures strata-vision-hip is compiled for (';'-separated)")
+if(STRATA_VISION_HIP AND STRATA_VISION_CUDA)
+ message(FATAL_ERROR "strata-vision: STRATA_VISION_CUDA and STRATA_VISION_HIP are two different programs")
+endif()
+if(STRATA_VISION_HIP AND NOT STRATA_VISION_HIP_ARCHS)
+ message(FATAL_ERROR "strata-vision: STRATA_VISION_HIP needs STRATA_VISION_HIP_ARCHS, for example \"gfx1200;gfx1201\"")
+endif()
option(STRATA_PORTABLE "build for other PCs: AVX2 baseline, static MSVC runtime" ON)
if(STRATA_PORTABLE AND MSVC)
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded")
@@ -52,6 +61,13 @@ else()
endif()
set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE)
set(GGML_CUDA ${STRATA_VISION_CUDA} CACHE BOOL "" FORCE)
+set(GGML_HIP ${STRATA_VISION_HIP} CACHE BOOL "" FORCE)
+if(STRATA_VISION_HIP)
+ # the engine's HIP build says the same (cmake/hip_backend.cmake); ggml-hip reads AMDGPU_TARGETS / CMAKE_HIP_ARCHITECTURES
+ set(AMDGPU_TARGETS "${STRATA_VISION_HIP_ARCHS}" CACHE STRING "" FORCE)
+ set(GPU_TARGETS "${STRATA_VISION_HIP_ARCHS}" CACHE STRING "" FORCE)
+ set(CMAKE_HIP_ARCHITECTURES "${STRATA_VISION_HIP_ARCHS}" CACHE STRING "" FORCE)
+endif()
add_subdirectory(${LLAMA_DIR} llama EXCLUDE_FROM_ALL)
@@ -59,9 +75,26 @@ add_executable(strata-vision strata_vision.cpp)
target_link_libraries(strata-vision PRIVATE mtmd llama)
target_compile_features(strata-vision PRIVATE cxx_std_17)
set_target_properties(strata-vision PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+if(STRATA_VISION_HIP)
+ # beside strata (and, in a component, the CPU reader "strata-vision"): the GPU reader is its own program, so a PC where
+ # the HIP runtime does not load still has the CPU one to fall back on
+ set_target_properties(strata-vision PROPERTIES OUTPUT_NAME strata-vision-hip)
+ string(REPLACE ";" "," _hip_archs "${STRATA_VISION_HIP_ARCHS}")
+ target_compile_definitions(strata-vision PRIVATE STRATA_VISION_HIP STRATA_VISION_HIP_ARCHS="${_hip_archs}")
+endif()
+if(UNIX AND NOT APPLE)
+ # the libraries it needs are the ones in its own folder (a component folder is relocatable): RUNPATH $ORIGIN
+ set_target_properties(strata-vision PROPERTIES BUILD_WITH_INSTALL_RPATH ON INSTALL_RPATH "$ORIGIN")
+ target_link_options(strata-vision PRIVATE -Wl,--enable-new-dtags)
+endif()
# opens a GGUF that has tensor types llama.cpp does not know (a Gyro quantization's TQK6 / TQK7) for its vocabulary only
add_executable(strata-vision-gguf-test EXCLUDE_FROM_ALL gguf_meta_test.cpp)
target_link_libraries(strata-vision-gguf-test PRIVATE llama)
target_compile_features(strata-vision-gguf-test PRIVATE cxx_std_17)
set_target_properties(strata-vision-gguf-test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+
+# the HIP reader's architecture check (gpu_arch.h), without a GPU
+add_executable(strata-vision-arch-test EXCLUDE_FROM_ALL gpu_arch_test.cpp)
+target_compile_features(strata-vision-arch-test PRIVATE cxx_std_17)
+set_target_properties(strata-vision-arch-test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
diff --git a/tools/vision/gpu_arch.h b/tools/vision/gpu_arch.h
new file mode 100644
index 000000000..411f52018
--- /dev/null
+++ b/tools/vision/gpu_arch.h
@@ -0,0 +1,33 @@
+// tools/vision/gpu_arch.h - which GPU the HIP picture reader (strata-vision-hip) was compiled for, and whether the GPU it
+// found is one of them. ggml-hip prints " Device 0: AMD Radeon RX 9070 XT, gfx1201:sramecc-:xnack- (0x1201), ..." when it
+// starts; a card whose architecture is not in the build would fail at its first kernel ("no kernel image", an abort), so
+// the reader reads that line and refuses before any kernel runs. The server then starts the CPU reader (see
+// serve/server.py, Vision._start). Header only, so gpu_arch_test.cpp can check it without a GPU.
+#pragma once
+
+#include <cctype>
+#include <string>
+#include <vector>
+
+// "gfx1201" out of ggml's device line (or a gcnArchName: "gfx1201:sramecc-:xnack-"); "" when there is none
+inline std::string strata_gfx_in(const std::string& text) {
+ for (size_t at = text.find("gfx"); at != std::string::npos; at = text.find("gfx", at + 1)) {
+ size_t end = at + 3;
+ while (end < text.size() && std::isxdigit((unsigned char) text[end])) ++end;
+ if (end > at + 3) return text.substr(at, end - at);
+ }
+ return "";
+}
+
+// "gfx1200,gfx1201" (the build's list; ',' or ';' between) holds `arch`
+inline bool strata_arch_listed(const std::string& list, const std::string& arch) {
+ if (arch.empty()) return false;
+ size_t start = 0;
+ while (start <= list.size()) {
+ size_t end = list.find_first_of(",; ", start);
+ if (end == std::string::npos) end = list.size();
+ if (list.compare(start, end - start, arch) == 0 && end - start == arch.size()) return true;
+ start = end + 1;
+ }
+ return false;
+}
diff --git a/tools/vision/gpu_arch_test.cpp b/tools/vision/gpu_arch_test.cpp
new file mode 100644
index 000000000..7cced4325
--- /dev/null
+++ b/tools/vision/gpu_arch_test.cpp
@@ -0,0 +1,24 @@
+// tools/vision/gpu_arch_test.cpp - the HIP picture reader's architecture check (gpu_arch.h) without a GPU. Exit 0 = passed.
+#include "gpu_arch.h"
+
+#include <cstdio>
+
+static int bad = 0;
+#define CHECK(c) do { if (!(c)) { std::printf("FAIL line %d: %s\n", __LINE__, #c); ++bad; } } while (0)
+
+int main() {
+ CHECK(strata_gfx_in(" Device 0: AMD Radeon RX 9070 XT, gfx1201:sramecc-:xnack- (0x1201), VMM: no, Wave Size: 32") == "gfx1201");
+ CHECK(strata_gfx_in(" Device 0: AMD Radeon Graphics, gfx1151 (0x1151)") == "gfx1151");
+ CHECK(strata_gfx_in(" Device 0: gfx, nothing after it") == "");
+ CHECK(strata_gfx_in("no architecture here") == "");
+ CHECK(strata_arch_listed("gfx1200,gfx1201", "gfx1201"));
+ CHECK(strata_arch_listed("gfx1200;gfx1201", "gfx1200"));
+ CHECK(strata_arch_listed("gfx1100", "gfx1100"));
+ CHECK(!strata_arch_listed("gfx1200,gfx1201", "gfx1100"));
+ CHECK(!strata_arch_listed("gfx1100", "gfx110")); // a prefix is not an architecture
+ CHECK(!strata_arch_listed("gfx1100", "gfx11000"));
+ CHECK(!strata_arch_listed("", "gfx1100"));
+ CHECK(!strata_arch_listed("gfx1100", ""));
+ std::printf(bad ? "gpu_arch_test: %d FAILED\n" : "gpu_arch_test: ok\n", bad);
+ return bad ? 1 : 0;
+}
diff --git a/tools/vision/strata_vision.cpp b/tools/vision/strata_vision.cpp
index 410d93024..b4f4582e8 100644
--- a/tools/vision/strata_vision.cpp
+++ b/tools/vision/strata_vision.cpp
@@ -13,9 +13,11 @@
// QUIT
// The output file is int32 {0x31455653 'SVE1', n_tokens, nx, ny, n_embd} then float32 [n_tokens][n_embd],
// row i at grid position (x = i % nx, y = i / nx). The text model is opened vocab-only (no weights).
+#include "ggml-backend.h"
#include "ggml-cpu.h"
#include "gguf.h"
#include "gguf_meta.h"
+#include "gpu_arch.h"
#include "llama.h"
#include "mtmd.h"
#include "mtmd-helper.h"
@@ -37,7 +39,14 @@
namespace {
+// the GPU architectures ggml-hip printed while it started (" Device 0: ..., gfx1201:sramecc-:xnack- (0x1201), ...")
+std::vector<std::string> g_gfx;
+
void quiet_log(ggml_log_level level, const char* text, void*) {
+ if (level == GGML_LOG_LEVEL_INFO && std::strstr(text, "Device ") != nullptr) {
+ const std::string gfx = strata_gfx_in(text);
+ if (!gfx.empty()) g_gfx.push_back(gfx);
+ }
if (level >= GGML_LOG_LEVEL_WARN) std::fputs(text, stderr);
}
@@ -70,6 +79,16 @@ int main(int argc, char** argv) {
if (a == "--mmproj") mmproj = next();
else if (a == "--model") model = next();
else if (a == "--gpu") gpu = true;
+ else if (a == "--version") {
+#ifdef STRATA_VISION_HIP
+ std::printf("strata-vision hip (GPU architectures: %s)
+", STRATA_VISION_HIP_ARCHS);
+#else
+ std::printf("strata-vision
+");
+#endif
+ return 0;
+ }
else if (a == "--threads") threads = std::atoi(next().c_str());
else if (a == "--max-tokens") max_tokens = std::atoi(next().c_str());
else if (a == "--min-tokens") min_tokens = std::atoi(next().c_str()); // mtmd image_min_tokens (#767)
@@ -91,13 +110,48 @@ int main(int argc, char** argv) {
if (!gpu) {
#ifdef _WIN32
_putenv_s("CUDA_VISIBLE_DEVICES", "-1");
+ _putenv_s("HIP_VISIBLE_DEVICES", "-1");
#else
setenv("CUDA_VISIBLE_DEVICES", "-1", 1);
+ setenv("HIP_VISIBLE_DEVICES", "-1", 1);
#endif
}
llama_log_set(quiet_log, nullptr);
mtmd_helper_log_set(quiet_log, nullptr);
llama_backend_init();
+ if (gpu) {
+ // Starting the backends prints the GPU line; with --gpu and no GPU, mtmd would quietly use the CPU, and a card
+ // the HIP build has no code for would abort at its first kernel. Both end here with a line the server reads
+ // (it then starts the CPU reader).
+ bool have_gpu = false;
+ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
+ const auto type = ggml_backend_dev_type(ggml_backend_dev_get(i));
+ have_gpu = have_gpu || type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU;
+ }
+ if (!have_gpu) {
+ std::printf("ERR this picture reader found no GPU (it was built for: %s)
+",
+#ifdef STRATA_VISION_HIP
+ STRATA_VISION_HIP_ARCHS
+#else
+ "none"
+#endif
+ );
+ std::fflush(stdout);
+ return 3;
+ }
+#ifdef STRATA_VISION_HIP
+ const char* check = std::getenv("STRATA_VISION_ARCH_CHECK");
+ if (!g_gfx.empty() && (check == nullptr || std::atoi(check) != 0) &&
+ !strata_arch_listed(STRATA_VISION_HIP_ARCHS, g_gfx.front())) {
+ std::printf("ERR this GPU (%s) is not in the picture reader's build (%s)
+", g_gfx.front().c_str(),
+ STRATA_VISION_HIP_ARCHS);
+ std::fflush(stdout);
+ return 3;
+ }
+#endif
+ }
const auto load_t0 = std::chrono::steady_clock::now();
#if !defined(_WIN32) && !defined(__APPLE__) // macOS has no posix_fadvise