Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

generate: spec_loop_runs() names the speculative loop's start rule (+ test); STRATA_SPEC_WATCH=<s>, a stall watchdog with the stage names for the CLI's verify loop (stall report and thread dump, off by default)

Niko12214c3edf47653c
diff --git a/CMakeLists.txt b/CMakeLists.txt
index fa275e8be..be5d7945d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1422,6 +1422,9 @@ if(STRATA_BUILD_TESTS)
add_executable(draft_policy_test src/spec/draft_policy_test.cpp)
target_link_libraries(draft_policy_test PRIVATE strata_spec)
add_test(NAME draft_policy_test COMMAND draft_policy_test)
+ add_executable(spec_start_test src/spec/spec_start_test.cpp)
+ target_include_directories(spec_start_test PRIVATE include)
+ add_test(NAME spec_start_test COMMAND spec_start_test)
endif()
if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)
add_executable(message_boundary_test src/program/message_boundary_test.cpp)
diff --git a/include/strata/spec/spec_start.hpp b/include/strata/spec/spec_start.hpp
new file mode 100644
index 000000000..94ee5c12a
--- /dev/null
+++ b/include/strata/spec/spec_start.hpp
@@ -0,0 +1,20 @@
+// include/strata/spec/spec_start.hpp - where the speculative loop of `strata generate --spec` begins, and whether it runs.
+//
+// The CLI's prompt path leaves the position the speculative loop starts at in one variable. Until 0.2.2 it was 0 for "the
+// loop is not used", which is also the position a native pack starts the loop at when the prompt is a single token (its
+// last prompt token is window 0 at position 0): such a prompt never speculated. -1 is the "not used" value now.
+#pragma once
+
+#include <cstdint>
+
+namespace strata::spec {
+
+inline constexpr int64_t kSpecNotStarted = -1;
+
+/// The loop runs when the prompt path named a start position (0 is a real one), tokens are still wanted and the text did
+/// not already end.
+inline constexpr bool spec_loop_runs(int64_t spec_pos, int64_t produced, int64_t max_new, bool ended) {
+ return spec_pos >= 0 && produced < max_new && !ended;
+}
+
+} // namespace strata::spec
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 482459957..dcfa71ad8 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -83,6 +83,7 @@
#include "strata/spec/draft_policy.hpp"
#include "strata/spec/draft_source.hpp"
#include "strata/spec/suffix_drafter.hpp"
+#include "strata/spec/spec_start.hpp"
#include "strata/program/batch_groups.hpp"
#include "strata/kernels/cvec.hpp"
#include "strata/core/progress.hpp"
@@ -15482,7 +15483,7 @@ int main(int argc, char** argv) {
// ---- plan v0.3 P5: the prompt's conditioning positions [0, n_prompt - 1) in batched chunks. The token loop
// then starts at the last prompt position, whose prediction is the first generated token.
int64_t pos_start = 0;
- int64_t spec_pos = -1; // plan v0.3 P6: where the speculative loop starts (-1 = not used; 0 is a real position: a 1-token prompt on a native pack)
+ int64_t spec_pos = strata::spec::kSpecNotStarted; // plan v0.3 P6: where the speculative loop starts (0 is a real position: a 1-token prompt on a native pack)
strata::prefill::Prefill prefill;
// #1595: not by default when the cache holds every expert (nothing to take)
strata::prefill::Prefill::arm_cpu_share(!multi_gpu && !o.no_pool,
@@ -15872,7 +15873,7 @@ int main(int argc, char** argv) {
// round emits (accepted drafts + 1) tokens. `commit` keeps the state of the tokens that were emitted.
const bool ended = o.stop_eos && !produced.empty() &&
std::find(o.eos_ids.begin(), o.eos_ids.end(), (int64_t) produced.back()) != o.eos_ids.end();
- if (spec_pos >= 0 && (int64_t) produced.size() < o.max_new && !ended) {
+ if (strata::spec::spec_loop_runs(spec_pos, (int64_t) produced.size(), o.max_new, ended)) {
auto read_ids = [](const std::string& path, std::vector<int64_t>& ids) {
std::ifstream in(path);
std::string text((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
@@ -16173,8 +16174,32 @@ int main(int argc, char** argv) {
}
const double pool_ms0 = drive.cpu_ms;
const int64_t misses0 = drive.d.multi_misses, entries0 = drive.d.multi_entries;
+ // STRATA_SPEC_WATCH=<seconds> (diagnostics, off by default): the serve watchdog for this loop. A run whose verify
+ // windows and prompt chunks stop beating for that long prints the stage it is in and the stall report, then ends.
+ if (const char* sw = std::getenv("STRATA_SPEC_WATCH"); sw != nullptr && std::atoi(sw) > 0) {
+ const int limit = std::atoi(sw);
+ strata::core::progress().busy.store(true);
+ std::thread([limit] {
+ strata::core::Progress& pr = strata::core::progress();
+ uint64_t last = pr.beats.load(), ticks_at = pr.ticks.load();
+ auto since = std::chrono::steady_clock::now();
+ for (;;) {
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ const auto now = std::chrono::steady_clock::now();
+ const uint64_t b = pr.beats.load();
+ if (b != last) { last = b; ticks_at = pr.ticks.load(); since = now; continue; }
+ if (now - since < std::chrono::seconds(limit)) continue;
+ std::fprintf(stderr, "strata generate: no progress for %d s (%s)\n", limit, stage_text().c_str());
+ stall_report(stderr, pr.ticks.load() - ticks_at);
+ strata::core::release_gpu_waits(stderr);
+ std::fflush(stderr);
+ std::abort();
+ }
+ }).detach();
+ }
while ((int64_t) produced.size() < max_new) {
const Clock::time_point t0 = Clock::now();
+ strata::core::progress_at("spec loop: choosing the window", (int64_t) produced.size());
int T = S_mtp;
if (use_mtp && o.spec_min_p > 0.0) {
T = 1;
@@ -16286,6 +16311,7 @@ int main(int argc, char** argv) {
bool adapt_ok = true;
if (!drive.d.usage.empty() && ((rounds + 1) % o.adapt_every) == 0)
adapt_thr = std::thread([&] { strata::platform::name_this_thread("strata-adapt"); strata::aux_cpus::pin_current_thread(); adapt_ok = adapt(); });
+ strata::core::progress_at("spec loop: commit", (int64_t) T);
if (!ver.commit(a + 1, err)) {
if (adapt_thr.joinable()) adapt_thr.join();
std::fprintf(stderr, "strata generate: %s\n", err.c_str());
diff --git a/src/spec/spec_start_test.cpp b/src/spec/spec_start_test.cpp
new file mode 100644
index 000000000..c046c8973
--- /dev/null
+++ b/src/spec/spec_start_test.cpp
@@ -0,0 +1,22 @@
+// src/spec/spec_start_test.cpp - the speculative loop's start position: 0 is a real position (a 1-token prompt on a native pack).
+#include "strata/spec/spec_start.hpp"
+
+#include <cstdio>
+
+using namespace strata::spec;
+
+int main() {
+ int fail = 0;
+ auto check = [&](bool ok, const char* what) {
+ if (!ok) { std::printf("FAIL: %s\n", what); ++fail; }
+ };
+ check(!spec_loop_runs(kSpecNotStarted, 0, 100, false), "no start position: the loop is not used");
+ check(spec_loop_runs(0, 0, 100, false), "position 0 (a 1-token prompt on a native pack) starts the loop");
+ check(spec_loop_runs(910, 0, 100, false), "a long prompt's start position");
+ check(spec_loop_runs(1, 0, 100, false), "position 1 (a 1-token prompt on the per-token path)");
+ check(!spec_loop_runs(0, 100, 100, false), "nothing left to generate");
+ check(!spec_loop_runs(0, 3, 100, true), "the text already ended");
+ check(kSpecNotStarted < 0, "the sentinel is not a position");
+ std::printf("spec_start tests: %s\n", fail ? "FAILED" : "OK");
+ return fail ? 1 : 0;
+}