Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(java): the shortlist fixture recorded floats at a precision BLAS can move

Adeshina Lasisi3af49cdf3a00
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index f3a5c0c5..fde7e5af 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -109,7 +109,21 @@ jobs:
# the stale committed copy -- exactly the failure the fixtures exist to prevent. The
# generator already leaves checkpoint-backed families alone and reports them unverified,
# so a bare --check is the right scope.
- run: python laya-java/scripts/gen_fixtures.py --check
+ #
+ # On failure, SHOW the drift. `--check` printed only "<family> is stale", which named the
+ # file and nothing else -- and diagnosing a cross-platform float difference from that took
+ # three CI round trips. Regenerating and diffing costs nothing on the passing path, which
+ # is the common one.
+ run: |
+ if python laya-java/scripts/gen_fixtures.py --check; then
+ exit 0
+ fi
+ echo "::group::what actually drifted"
+ python laya-java/scripts/gen_fixtures.py || true
+ git --no-pager diff --stat -- laya-java/fixtures
+ git --no-pager diff -U0 -- laya-java/fixtures | head -120
+ echo "::endgroup::"
+ exit 1
# The compiled-in Unicode tables are recorded from CPython, so a CPython upgrade that moves
# them has to show up here rather than as a routing change nobody attributed.
- name: Unicode tables match CPython
diff --git a/laya-java/fixtures/shortlist.json b/laya-java/fixtures/shortlist.json
index ccfbe0f0..e6e60d4d 100644
--- a/laya-java/fixtures/shortlist.json
+++ b/laya-java/fixtures/shortlist.json
@@ -1,5 +1,5 @@
{
- "score_decimals": 12,
+ "score_decimals": 9,
"default_k": 20,
"embed_dim": 8,
"embed_probe": {
@@ -100,11 +100,11 @@
"card_delivery_estimate"
],
"scores": [
- 0.947468947226,
- 0.907542741113,
- 0.584010550099,
- 0.523549809346,
- 0.430750026379
+ 0.947468947,
+ 0.907542741,
+ 0.58401055,
+ 0.523549809,
+ 0.430750026
],
"passthrough": false,
"n": 17,
@@ -167,7 +167,7 @@
"wrong_amount_of_cash_received"
],
"scores": [
- 0.947468947226
+ 0.947468947
],
"passthrough": false,
"n": 17,
@@ -241,22 +241,22 @@
"lost_or_stolen_card"
],
"scores": [
- 0.947468947226,
- 0.907542741113,
- 0.584010550099,
- 0.523549809346,
- 0.430750026379,
- 0.416119355798,
- 0.367579134772,
- 0.298956810054,
- 0.196754725434,
- 0.132558511906,
- 0.107369708039,
- -0.133314503668,
- -0.164844834842,
- -0.247795808363,
- -0.920650595907,
- -0.947114785336
+ 0.947468947,
+ 0.907542741,
+ 0.58401055,
+ 0.523549809,
+ 0.430750026,
+ 0.416119356,
+ 0.367579135,
+ 0.29895681,
+ 0.196754725,
+ 0.132558512,
+ 0.107369708,
+ -0.133314504,
+ -0.164844835,
+ -0.247795808,
+ -0.920650596,
+ -0.947114785
],
"passthrough": false,
"n": 17,
@@ -445,8 +445,8 @@
"technical"
],
"scores": [
- 0.304439081979,
- -0.609827188998
+ 0.304439082,
+ -0.609827189
],
"passthrough": false,
"n": 3,
@@ -535,10 +535,10 @@
"pending_card_payment"
],
"scores": [
- 0.772003687973,
- 0.697649917343,
- 0.695222725058,
- 0.318564649233
+ 0.772003688,
+ 0.697649917,
+ 0.695222725,
+ 0.318564649
],
"passthrough": false,
"n": 17,
@@ -603,10 +603,10 @@
"direct_debit_payment_not_recognised"
],
"scores": [
- 0.939301271345,
- 0.882771141579,
- 0.468142800581,
- 0.46266025283
+ 0.939301271,
+ 0.882771142,
+ 0.468142801,
+ 0.462660253
],
"passthrough": false,
"n": 17,
@@ -671,10 +671,10 @@
"pending_card_payment"
],
"scores": [
- 0.696734262273,
- 0.61900567629,
- 0.612605418621,
- 0.359794006663
+ 0.696734262,
+ 0.619005676,
+ 0.612605419,
+ 0.359794007
],
"passthrough": false,
"n": 17,
@@ -739,10 +739,10 @@
"request_refund"
],
"scores": [
- 0.330675885316,
- 0.257026035401,
- 0.243526050327,
- 0.223691482192
+ 0.330675885,
+ 0.257026035,
+ 0.24352605,
+ 0.223691482
],
"passthrough": false,
"n": 17,
@@ -807,10 +807,10 @@
"failed_transfer"
],
"scores": [
- 0.986196757011,
- 0.936378299317,
- 0.484560069972,
- 0.44363985375
+ 0.986196757,
+ 0.936378299,
+ 0.48456007,
+ 0.443639854
],
"passthrough": false,
"n": 17,
@@ -875,10 +875,10 @@
"card_not_working"
],
"scores": [
- 0.947468947226,
- 0.907542741113,
- 0.584010550099,
- 0.523549809346
+ 0.947468947,
+ 0.907542741,
+ 0.58401055,
+ 0.523549809
],
"passthrough": false,
"n": 17,
@@ -915,8 +915,8 @@
"a"
],
"scores": [
- 0.304439081979,
- -0.123272321994
+ 0.304439082,
+ -0.123272322
],
"passthrough": false,
"n": 4,
@@ -951,8 +951,8 @@
"beta"
],
"scores": [
- 0.426002656466,
- 0.128698817888
+ 0.426002656,
+ 0.128698818
],
"passthrough": false,
"n": 4,
diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/ShortlistTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/ShortlistTest.java
index 3d1eacb5..df40657b 100644
--- a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/ShortlistTest.java
+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/ShortlistTest.java
@@ -90,13 +90,13 @@ class ShortlistTest {
* OpenBLAS blocks it differently again. So the reference's own score is not reproducible
* across machines and no port can match it bit for bit.
*
- * <p>The fixture therefore records twelve decimals, which is roughly four orders of magnitude
- * tighter than any porting error this repository has seen and four orders looser than the
+ * <p>The fixture therefore records nine decimals, which is roughly three orders of magnitude
+ * tighter than any porting error this repository has seen and seven orders looser than the
* 1e-16 BLAS spread. What is still compared exactly is the label ORDER -- the only thing a
* consumer observes, and the thing a ranking bug actually moves.
*/
private static void assertScore(double expected, double actual, String what) {
- double tolerance = 5e-13;
+ double tolerance = 5e-10;
assertTrue(Math.abs(expected - actual) <= tolerance,
what + ": expected " + expected + " +/- " + tolerance + ", got " + actual
+ " (difference " + Math.abs(expected - actual) + ")");
@@ -511,7 +511,7 @@ class ShortlistTest {
void underflowingQueryScoresExactlyZero() {
// Asserted exactly, and separately from the recorded scores, because the rounded
// comparison cannot see it: without the zero-norm guard the score is 1.4e-200, which
- // agrees with 0 to twelve decimals and to any tolerance worth having. A norm of zero does
+ // agrees with 0 to nine decimals and to any tolerance worth having. A norm of zero does
// NOT imply a dot product of zero once components underflow, which is the trap.
double[] query = {1e-200, 1e-200};
double[][] matrix = {query, {1.0, 1.0}};
@@ -524,7 +524,7 @@ class ShortlistTest {
void cosineIsClampedToOne() {
// The self-cosine of this vector computes to 1.0000000000000002 with sequential
// arithmetic, so the clamp is observable -- and again only exactly: the unclamped value
- // agrees with 1.0 to twelve decimals. Asserted on an exact equality for that reason.
+ // agrees with 1.0 to nine decimals. Asserted on an exact equality for that reason.
double[] vector = {1.0 / 3.0, 1.0 / 3.0, 8.0 / 3.0};
double[][] matrix = {vector, vector};
double[] sims = Shortlist.cosine(vector, matrix, 1);
@@ -838,13 +838,13 @@ class ShortlistTest {
@DisplayName("the recorded scores are rounded far below the tolerance they are checked at")
void scorePrecisionIsHonest() {
// Guards the one tolerance in this suite from quietly widening. The fixture rounds to
- // twelve decimals and the comparison allows 5e-13 -- half of the last recorded digit --
+ // nine decimals and the comparison allows 5e-10 -- half of the last recorded digit --
// so the tolerance admits nothing the rounding did not already discard. If either number
// moves without the other, this fails rather than letting a looser check pass unnoticed.
int decimals = ((Number) fixture().get("score_decimals")).intValue();
- assertEquals(12, decimals, "the fixture's recorded precision changed");
+ assertEquals(9, decimals, "the fixture's recorded precision changed");
double halfOfLastDigit = 0.5 * Math.pow(10, -decimals);
- assertEquals(5e-13, halfOfLastDigit, 1e-18,
+ assertEquals(5e-10, halfOfLastDigit, 1e-15,
"the tolerance in assertScore must stay half of the last recorded digit");
// and the recorded values really are rounded, not full precision stored at a width that
// happens to look rounded
@@ -859,7 +859,7 @@ class ShortlistTest {
continue;
}
// Via the shortest decimal that round-trips, not by scaling: multiplying a
- // 12-decimal value by 1e12 is not exact in binary, and the first version of this
+ // 9-decimal value by 1e9 is not exact in binary, and the first version of this
// check failed on its own arithmetic rather than on the data.
double value = ((Number) score).doubleValue();
int scale = new java.math.BigDecimal(Double.toString(value)).stripTrailingZeros()
diff --git a/laya-java/scripts/gen_fixtures.py b/laya-java/scripts/gen_fixtures.py
index 5bdc15e0..c5790af3 100644
--- a/laya-java/scripts/gen_fixtures.py
+++ b/laya-java/scripts/gen_fixtures.py
@@ -1114,7 +1114,18 @@ def shortlist():
# shown up in this repository, and about four orders LOOSER than the 1e-16 BLAS spread, so it
# separates the two cleanly. What stays exact is the LABEL ORDER, which is the only thing a
# consumer can observe -- and the thing a ranking bug actually changes.
-_SCORE_DECIMALS = 12
+# Nine, measured rather than chosen. A cosine here is a dot product of 8-dimensional unit
+# vectors, and BLAS blocking moves the last bit: the worst difference between numpy's `dot` and a
+# sequential sum over 4,000 random pairs is 2.22e-16, about one ULP near 1.
+#
+# A decimal snapshot of a platform-dependent float can always straddle a rounding boundary, so the
+# only question is how often. With 194 floats in this file the chance that at least one lands
+# within the BLAS error of a boundary is about 8.3% at twelve decimals -- which is what happened:
+# the drift gate failed on Linux x64 while reproducing exactly on arm64, with the same numpy,
+# torch and transformers pins. At nine it is 0.0085%, and nine is still three orders of magnitude
+# finer than any real porting bug: a wrong normalisation, a missing clamp or an inverted tie order
+# moves a score in the third decimal, not the tenth.
+_SCORE_DECIMALS = 9
def _cache_counter_cases():
@@ -1138,7 +1149,10 @@ def _cache_counter_cases():
def _round_score(value):
- """A cosine rounded to a precision BLAS blocking cannot move, or tagged when non-finite."""
+ """A cosine rounded to a precision BLAS blocking is unlikely to move, or tagged if non-finite.
+
+ NOT "cannot move" -- that is what this said, and it was wrong. See _SCORE_DECIMALS.
+ """
value = float(value)
if value != value or value in (float("inf"), float("-inf")):
return _json_float(value)