Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
strata-vision: HIP build (strata-vision-hip) for AMD GPUs; the reader refuses a GPU that is not in its build; RUNPATH $ORIGIN
Niko1221fb14c354b414
diff --git a/tools/vision/CMakeLists.txt b/tools/vision/CMakeLists.txtindex ed42e6254..5b09e2d6f 100644--- a/tools/vision/CMakeLists.txt+++ b/tools/vision/CMakeLists.txt@@ -3,6 +3,7 @@## cmake -S tools/vision -B build-vision -G Ninja -DCMAKE_BUILD_TYPE=Release -DLLAMA_DIR=third_party/llama.cpp# [-DSTRATA_VISION_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native] [-DSTRATA_PORTABLE=OFF]+# [-DSTRATA_VISION_HIP=ON -DSTRATA_VISION_HIP_ARCHS="gfx1200;gfx1201" + the HIP compiler the engine build uses]# cmake --build build-vision --target strata-vision## Portable (AVX2 baseline) is the default, so an encoder built without extra flags runs on any AVX2 PC: the 0.1.32@@ -14,6 +15,14 @@ project(strata_vision C CXX)set(LLAMA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../third_party/llama.cpp" CACHE PATH "llama.cpp checkout")option(STRATA_VISION_CUDA "run the vision encoder on the GPU (builds ggml-cuda)" OFF)+option(STRATA_VISION_HIP "run the vision encoder on an AMD GPU (builds ggml-hip); the program is named strata-vision-hip" OFF)+set(STRATA_VISION_HIP_ARCHS "" CACHE STRING "the gfx architectures strata-vision-hip is compiled for (';'-separated)")+if(STRATA_VISION_HIP AND STRATA_VISION_CUDA)+ message(FATAL_ERROR "strata-vision: STRATA_VISION_CUDA and STRATA_VISION_HIP are two different programs")+endif()+if(STRATA_VISION_HIP AND NOT STRATA_VISION_HIP_ARCHS)+ message(FATAL_ERROR "strata-vision: STRATA_VISION_HIP needs STRATA_VISION_HIP_ARCHS, for example \"gfx1200;gfx1201\"")+endif()option(STRATA_PORTABLE "build for other PCs: AVX2 baseline, static MSVC runtime" ON)if(STRATA_PORTABLE AND MSVC)set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded")@@ -52,6 +61,13 @@ else()endif()set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE)set(GGML_CUDA ${STRATA_VISION_CUDA} CACHE BOOL "" FORCE)+set(GGML_HIP ${STRATA_VISION_HIP} CACHE BOOL "" FORCE)+if(STRATA_VISION_HIP)+ # the engine's HIP build says the same (cmake/hip_backend.cmake); ggml-hip reads AMDGPU_TARGETS / CMAKE_HIP_ARCHITECTURES+ set(AMDGPU_TARGETS "${STRATA_VISION_HIP_ARCHS}" CACHE STRING "" FORCE)+ set(GPU_TARGETS "${STRATA_VISION_HIP_ARCHS}" CACHE STRING "" FORCE)+ set(CMAKE_HIP_ARCHITECTURES "${STRATA_VISION_HIP_ARCHS}" CACHE STRING "" FORCE)+endif()add_subdirectory(${LLAMA_DIR} llama EXCLUDE_FROM_ALL)@@ -59,9 +75,26 @@ add_executable(strata-vision strata_vision.cpp)target_link_libraries(strata-vision PRIVATE mtmd llama)target_compile_features(strata-vision PRIVATE cxx_std_17)set_target_properties(strata-vision PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)+if(STRATA_VISION_HIP)+ # beside strata (and, in a component, the CPU reader "strata-vision"): the GPU reader is its own program, so a PC where+ # the HIP runtime does not load still has the CPU one to fall back on+ set_target_properties(strata-vision PROPERTIES OUTPUT_NAME strata-vision-hip)+ string(REPLACE ";" "," _hip_archs "${STRATA_VISION_HIP_ARCHS}")+ target_compile_definitions(strata-vision PRIVATE STRATA_VISION_HIP STRATA_VISION_HIP_ARCHS="${_hip_archs}")+endif()+if(UNIX AND NOT APPLE)+ # the libraries it needs are the ones in its own folder (a component folder is relocatable): RUNPATH $ORIGIN+ set_target_properties(strata-vision PROPERTIES BUILD_WITH_INSTALL_RPATH ON INSTALL_RPATH "$ORIGIN")+ target_link_options(strata-vision PRIVATE -Wl,--enable-new-dtags)+endif()# opens a GGUF that has tensor types llama.cpp does not know (a Gyro quantization's TQK6 / TQK7) for its vocabulary onlyadd_executable(strata-vision-gguf-test EXCLUDE_FROM_ALL gguf_meta_test.cpp)target_link_libraries(strata-vision-gguf-test PRIVATE llama)target_compile_features(strata-vision-gguf-test PRIVATE cxx_std_17)set_target_properties(strata-vision-gguf-test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)++# the HIP reader's architecture check (gpu_arch.h), without a GPU+add_executable(strata-vision-arch-test EXCLUDE_FROM_ALL gpu_arch_test.cpp)+target_compile_features(strata-vision-arch-test PRIVATE cxx_std_17)+set_target_properties(strata-vision-arch-test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)diff --git a/tools/vision/gpu_arch.h b/tools/vision/gpu_arch.hnew file mode 100644index 000000000..411f52018--- /dev/null+++ b/tools/vision/gpu_arch.h@@ -0,0 +1,33 @@+// tools/vision/gpu_arch.h - which GPU the HIP picture reader (strata-vision-hip) was compiled for, and whether the GPU it+// found is one of them. ggml-hip prints " Device 0: AMD Radeon RX 9070 XT, gfx1201:sramecc-:xnack- (0x1201), ..." when it+// starts; a card whose architecture is not in the build would fail at its first kernel ("no kernel image", an abort), so+// the reader reads that line and refuses before any kernel runs. The server then starts the CPU reader (see+// serve/server.py, Vision._start). Header only, so gpu_arch_test.cpp can check it without a GPU.+#pragma once++#include <cctype>+#include <string>+#include <vector>++// "gfx1201" out of ggml's device line (or a gcnArchName: "gfx1201:sramecc-:xnack-"); "" when there is none+inline std::string strata_gfx_in(const std::string& text) {+ for (size_t at = text.find("gfx"); at != std::string::npos; at = text.find("gfx", at + 1)) {+ size_t end = at + 3;+ while (end < text.size() && std::isxdigit((unsigned char) text[end])) ++end;+ if (end > at + 3) return text.substr(at, end - at);+ }+ return "";+}++// "gfx1200,gfx1201" (the build's list; ',' or ';' between) holds `arch`+inline bool strata_arch_listed(const std::string& list, const std::string& arch) {+ if (arch.empty()) return false;+ size_t start = 0;+ while (start <= list.size()) {+ size_t end = list.find_first_of(",; ", start);+ if (end == std::string::npos) end = list.size();+ if (list.compare(start, end - start, arch) == 0 && end - start == arch.size()) return true;+ start = end + 1;+ }+ return false;+}diff --git a/tools/vision/gpu_arch_test.cpp b/tools/vision/gpu_arch_test.cppnew file mode 100644index 000000000..7cced4325--- /dev/null+++ b/tools/vision/gpu_arch_test.cpp@@ -0,0 +1,24 @@+// tools/vision/gpu_arch_test.cpp - the HIP picture reader's architecture check (gpu_arch.h) without a GPU. Exit 0 = passed.+#include "gpu_arch.h"++#include <cstdio>++static int bad = 0;+#define CHECK(c) do { if (!(c)) { std::printf("FAIL line %d: %s\n", __LINE__, #c); ++bad; } } while (0)++int main() {+ CHECK(strata_gfx_in(" Device 0: AMD Radeon RX 9070 XT, gfx1201:sramecc-:xnack- (0x1201), VMM: no, Wave Size: 32") == "gfx1201");+ CHECK(strata_gfx_in(" Device 0: AMD Radeon Graphics, gfx1151 (0x1151)") == "gfx1151");+ CHECK(strata_gfx_in(" Device 0: gfx, nothing after it") == "");+ CHECK(strata_gfx_in("no architecture here") == "");+ CHECK(strata_arch_listed("gfx1200,gfx1201", "gfx1201"));+ CHECK(strata_arch_listed("gfx1200;gfx1201", "gfx1200"));+ CHECK(strata_arch_listed("gfx1100", "gfx1100"));+ CHECK(!strata_arch_listed("gfx1200,gfx1201", "gfx1100"));+ CHECK(!strata_arch_listed("gfx1100", "gfx110")); // a prefix is not an architecture+ CHECK(!strata_arch_listed("gfx1100", "gfx11000"));+ CHECK(!strata_arch_listed("", "gfx1100"));+ CHECK(!strata_arch_listed("gfx1100", ""));+ std::printf(bad ? "gpu_arch_test: %d FAILED\n" : "gpu_arch_test: ok\n", bad);+ return bad ? 1 : 0;+}diff --git a/tools/vision/strata_vision.cpp b/tools/vision/strata_vision.cppindex 410d93024..b4f4582e8 100644--- a/tools/vision/strata_vision.cpp+++ b/tools/vision/strata_vision.cpp@@ -13,9 +13,11 @@// QUIT// The output file is int32 {0x31455653 'SVE1', n_tokens, nx, ny, n_embd} then float32 [n_tokens][n_embd],// row i at grid position (x = i % nx, y = i / nx). The text model is opened vocab-only (no weights).+#include "ggml-backend.h"#include "ggml-cpu.h"#include "gguf.h"#include "gguf_meta.h"+#include "gpu_arch.h"#include "llama.h"#include "mtmd.h"#include "mtmd-helper.h"@@ -37,7 +39,14 @@namespace {+// the GPU architectures ggml-hip printed while it started (" Device 0: ..., gfx1201:sramecc-:xnack- (0x1201), ...")+std::vector<std::string> g_gfx;+void quiet_log(ggml_log_level level, const char* text, void*) {+ if (level == GGML_LOG_LEVEL_INFO && std::strstr(text, "Device ") != nullptr) {+ const std::string gfx = strata_gfx_in(text);+ if (!gfx.empty()) g_gfx.push_back(gfx);+ }if (level >= GGML_LOG_LEVEL_WARN) std::fputs(text, stderr);}@@ -70,6 +79,16 @@ int main(int argc, char** argv) {if (a == "--mmproj") mmproj = next();else if (a == "--model") model = next();else if (a == "--gpu") gpu = true;+ else if (a == "--version") {+#ifdef STRATA_VISION_HIP+ std::printf("strata-vision hip (GPU architectures: %s)+", STRATA_VISION_HIP_ARCHS);+#else+ std::printf("strata-vision+");+#endif+ return 0;+ }else if (a == "--threads") threads = std::atoi(next().c_str());else if (a == "--max-tokens") max_tokens = std::atoi(next().c_str());else if (a == "--min-tokens") min_tokens = std::atoi(next().c_str()); // mtmd image_min_tokens (#767)@@ -91,13 +110,48 @@ int main(int argc, char** argv) {if (!gpu) {#ifdef _WIN32_putenv_s("CUDA_VISIBLE_DEVICES", "-1");+ _putenv_s("HIP_VISIBLE_DEVICES", "-1");#elsesetenv("CUDA_VISIBLE_DEVICES", "-1", 1);+ setenv("HIP_VISIBLE_DEVICES", "-1", 1);#endif}llama_log_set(quiet_log, nullptr);mtmd_helper_log_set(quiet_log, nullptr);llama_backend_init();+ if (gpu) {+ // Starting the backends prints the GPU line; with --gpu and no GPU, mtmd would quietly use the CPU, and a card+ // the HIP build has no code for would abort at its first kernel. Both end here with a line the server reads+ // (it then starts the CPU reader).+ bool have_gpu = false;+ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {+ const auto type = ggml_backend_dev_type(ggml_backend_dev_get(i));+ have_gpu = have_gpu || type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU;+ }+ if (!have_gpu) {+ std::printf("ERR this picture reader found no GPU (it was built for: %s)+",+#ifdef STRATA_VISION_HIP+ STRATA_VISION_HIP_ARCHS+#else+ "none"+#endif+ );+ std::fflush(stdout);+ return 3;+ }+#ifdef STRATA_VISION_HIP+ const char* check = std::getenv("STRATA_VISION_ARCH_CHECK");+ if (!g_gfx.empty() && (check == nullptr || std::atoi(check) != 0) &&+ !strata_arch_listed(STRATA_VISION_HIP_ARCHS, g_gfx.front())) {+ std::printf("ERR this GPU (%s) is not in the picture reader's build (%s)+", g_gfx.front().c_str(),+ STRATA_VISION_HIP_ARCHS);+ std::fflush(stdout);+ return 3;+ }+#endif+ }const auto load_t0 = std::chrono::steady_clock::now();#if !defined(_WIN32) && !defined(__APPLE__) // macOS has no posix_fadvise