Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tests: glm53's pin-branch harness compares within 1e-5 through the Vulkan chain

JustVugg180cdae9e529
diff --git a/c/tests/glm53_pin_branch_harness.py b/c/tests/glm53_pin_branch_harness.py
index 5f2df89e..783d09cc 100644
--- a/c/tests/glm53_pin_branch_harness.py
+++ b/c/tests/glm53_pin_branch_harness.py
@@ -86,10 +86,22 @@ def tail_scores(binary, fixture, branch_in_between):
return scores
+def same(a, b, tol):
+ if tol <= 0 or a is None:
+ return a == b
+ return abs(float(a) - float(b)) <= tol
+
+
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--binary", required=True)
parser.add_argument("--fixture", type=Path, required=True)
+ parser.add_argument("--tol", type=float, default=0.0,
+ help="largest |difference| allowed between a score and the cold engine's "
+ "(0: the same text). The Vulkan chain computes a long prompt with its "
+ "blocked attention and a short tail with the per-row one: they agree "
+ "to rounding, and a pin restored over rows that are not its own is "
+ "off by far more")
arguments = parser.parse_args()
if not (arguments.fixture / "config.json").exists():
print(f"SKIP: manca {arguments.fixture}; generala con\n"
@@ -109,7 +121,7 @@ def main() -> int:
for branch_in_between in (False, True):
scores = tail_scores(binary, arguments.fixture, branch_in_between)
- wrong = [p for p in tail if scores.get(p) != fresh[p]]
+ wrong = [p for p in tail if not same(scores.get(p), fresh[p], arguments.tol)]
if wrong:
where = "con un ramo P+B in mezzo" if branch_in_between else "senza rami"
print(f"FAIL: {where}, {len(wrong)} posizioni su {len(tail)} della coda "
diff --git a/c/tests/vulkan_dense_only_glm.sh b/c/tests/vulkan_dense_only_glm.sh
index 82494d46..b97c7554 100644
--- a/c/tests/vulkan_dense_only_glm.sh
+++ b/c/tests/vulkan_dense_only_glm.sh
@@ -194,7 +194,7 @@ dho_family_glm() {
CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32 KV_SLOTS=2
COLI_VK_CHAIN=2 CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32
unset CHAIN_SERVE_EXPECT
- COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_USAGE=$PWD/chain.usage COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 $PY tests/glm53_pin_branch_harness.py --binary ./glm53 --fixture glm53_mm_tiny
+ COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_USAGE=$PWD/chain.usage COLI_VK_TIER=0 $PY tests/glm53_pin_branch_harness.py --binary ./glm53 --fixture glm53_mm_tiny --tol 1e-5
unset COLI_VK_DENSE_HOST
# the expert cache sizes itself from the free memory after the host copies went (on a
# device that shares the RAM the device's copy takes the same RAM: no gain to show there;
diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.sh
index 606a4d11..8b877aaa 100755
--- a/c/tests/vulkan_engines.sh
+++ b/c/tests/vulkan_engines.sh
@@ -2124,9 +2124,9 @@ family_glm_chain() {
CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32 KV_SLOTS=2
COLI_VK_CHAIN=2 CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32
# a pin restored over rows another branch rewrote: the KDA state goes up from the pin,
- # the MLA rows' watermark comes down (the tier off: two engine processes compared bit
- # for bit, see vulkan_partial_glm.sh)
- COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_USAGE=$PWD/chain.usage COLI_VK_TIER=0 $PY tests/glm53_pin_branch_harness.py --binary ./glm53 --fixture glm53_mm_tiny
+ # the MLA rows' watermark comes down (the tier off and the scores within 1e-5: two
+ # engine processes, see vulkan_partial_glm.sh)
+ COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_USAGE=$PWD/chain.usage COLI_VK_TIER=0 $PY tests/glm53_pin_branch_harness.py --binary ./glm53 --fixture glm53_mm_tiny --tol 1e-5
unset OMP_NUM_THREADS CAP_RAISE
}
@@ -3717,7 +3717,7 @@ kv_split_glm53() {
CHAIN_SERVE_EXPECT="$E" CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32 $K
CHAIN_SERVE_EXPECT="$E" CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=4 COLI_VK_CHAIN_ROWS=3 $K
CHAIN_SERVE_EXPECT="$E" CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32 KV_SLOTS=2 $K
- env $K COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_USAGE=$PWD/chain.usage COLI_VK_TIER=0 $PY tests/glm53_pin_branch_harness.py --binary ./glm53 --fixture glm53_mm_tiny
+ env $K COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_USAGE=$PWD/chain.usage COLI_VK_TIER=0 $PY tests/glm53_pin_branch_harness.py --binary ./glm53 --fixture glm53_mm_tiny --tol 1e-5
}
}
kv_split_glm53_sanitize() {
diff --git a/c/tests/vulkan_partial_glm.sh b/c/tests/vulkan_partial_glm.sh
index ed57ee4f..41509c2b 100644
--- a/c/tests/vulkan_partial_glm.sh
+++ b/c/tests/vulkan_partial_glm.sh
@@ -377,10 +377,11 @@ ptl_family_glm() {
CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_l6_serve GLM53_BITS=32 COLI_VK_CHAIN_LAYERS=3
unset CHAIN_SERVE_EXPECT
# a pin restored over rows another branch rewrote, 2 of 4 layers on the device. The
- # harness compares two engine processes bit for bit, so the tier is off: which experts
- # a device computes depends on each process's history, and a device expert's f32 sum
- # can differ from the CPU's in its last bit (CI's runners did, at one position).
- COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_CHAIN_LAYERS=2 COLI_USAGE=$PWD/chain.usage COLI_VK_TIER=0 $PY tests/glm53_pin_branch_harness.py --binary ./glm53 --fixture glm53_mm_tiny
+ # harness compares two engine processes, so the tier is off (which experts a device
+ # computes depends on each process's history), and the scores within 1e-5: the cold
+ # engine's prompt takes the blocked attention, the warm one's short tail the per-row
+ # one, which agree to rounding (CI's runners differed at one position by 1e-6).
+ COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_CHAIN_LAYERS=2 COLI_USAGE=$PWD/chain.usage COLI_VK_TIER=0 $PY tests/glm53_pin_branch_harness.py --binary ./glm53 --fixture glm53_mm_tiny --tol 1e-5
rm -rf glm_tiny_shx glm53_l6 glm53_l6s-i4 glm53_l6_serve chain.usage
rm -f ptl-probe.err ptl-ram*.log same-*.log same-*.tok same-*.f32
unset OMP_NUM_THREADS CAP_RAISE