Apache Lucene open-source search software
simple cleanups to vector code (#12680)
Robert Muirdb746bdbc398
diff --git a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.javaindex c066304136f..1b7157c4752 100644--- a/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java+++ b/lucene/core/src/java20/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java@@ -16,12 +16,15 @@*/package org.apache.lucene.internal.vectorization;+import static jdk.incubator.vector.VectorOperators.ADD;+import static jdk.incubator.vector.VectorOperators.B2S;+import static jdk.incubator.vector.VectorOperators.S2I;+import jdk.incubator.vector.ByteVector;import jdk.incubator.vector.FloatVector;import jdk.incubator.vector.IntVector;import jdk.incubator.vector.ShortVector;import jdk.incubator.vector.Vector;-import jdk.incubator.vector.VectorOperators;import jdk.incubator.vector.VectorShape;import jdk.incubator.vector.VectorSpecies;@@ -29,21 +32,25 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {private static final int INT_SPECIES_PREF_BIT_SIZE = IntVector.SPECIES_PREFERRED.vectorBitSize();- private static final VectorSpecies<Float> PREF_FLOAT_SPECIES = FloatVector.SPECIES_PREFERRED;- private static final VectorSpecies<Byte> PREF_BYTE_SPECIES;- private static final VectorSpecies<Short> PREF_SHORT_SPECIES;+ // we always use the platform's maximum floating point vector size+ private static final VectorSpecies<Float> FLOAT_SPECIES = FloatVector.SPECIES_PREFERRED;++ // for integer methods, it is more complicated due to conversions+ private static final VectorSpecies<Byte> BYTE_SPECIES;+ private static final VectorSpecies<Short> SHORT_SPECIES;+ // compute BYTE/SHORT sizes relative to preferred integer vector sizestatic {if (INT_SPECIES_PREF_BIT_SIZE >= 256) {- PREF_BYTE_SPECIES =+ BYTE_SPECIES =ByteVector.SPECIES_MAX.withShape(VectorShape.forBitSize(IntVector.SPECIES_PREFERRED.vectorBitSize() >> 2));- PREF_SHORT_SPECIES =+ SHORT_SPECIES =ShortVector.SPECIES_MAX.withShape(VectorShape.forBitSize(IntVector.SPECIES_PREFERRED.vectorBitSize() >> 1));} else {- PREF_BYTE_SPECIES = null;- PREF_SHORT_SPECIES = null;+ BYTE_SPECIES = null;+ SHORT_SPECIES = null;}}@@ -57,124 +64,80 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {public float dotProduct(float[] a, float[] b) {int i = 0;float res = 0;+// if the array size is large (> 2x platform vector size), its worth the overhead to vectorize- if (a.length > 2 * PREF_FLOAT_SPECIES.length()) {- // vector loop is unrolled 4x (4 accumulators in parallel)- FloatVector acc1 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector acc2 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector acc3 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector acc4 = FloatVector.zero(PREF_FLOAT_SPECIES);- int upperBound = PREF_FLOAT_SPECIES.loopBound(a.length - 3 * PREF_FLOAT_SPECIES.length());- for (; i < upperBound; i += 4 * PREF_FLOAT_SPECIES.length()) {- FloatVector va = FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i);- FloatVector vb = FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i);- acc1 = acc1.add(va.mul(vb));- FloatVector vc =- FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i + PREF_FLOAT_SPECIES.length());- FloatVector vd =- FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i + PREF_FLOAT_SPECIES.length());- acc2 = acc2.add(vc.mul(vd));- FloatVector ve =- FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i + 2 * PREF_FLOAT_SPECIES.length());- FloatVector vf =- FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i + 2 * PREF_FLOAT_SPECIES.length());- acc3 = acc3.add(ve.mul(vf));- FloatVector vg =- FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i + 3 * PREF_FLOAT_SPECIES.length());- FloatVector vh =- FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i + 3 * PREF_FLOAT_SPECIES.length());- acc4 = acc4.add(vg.mul(vh));- }- // vector tail: less scalar computations for unaligned sizes, esp with big vector sizes- upperBound = PREF_FLOAT_SPECIES.loopBound(a.length);- for (; i < upperBound; i += PREF_FLOAT_SPECIES.length()) {- FloatVector va = FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i);- FloatVector vb = FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i);- acc1 = acc1.add(va.mul(vb));- }- // reduce- FloatVector res1 = acc1.add(acc2);- FloatVector res2 = acc3.add(acc4);- res += res1.add(res2).reduceLanes(VectorOperators.ADD);+ if (a.length > 2 * FLOAT_SPECIES.length()) {+ i += FLOAT_SPECIES.loopBound(a.length);+ res += dotProductBody(a, b, i);}+ // scalar tailfor (; i < a.length; i++) {res += b[i] * a[i];}return res;}+ /** vectorized float dot product body */+ private float dotProductBody(float[] a, float[] b, int limit) {+ int i = 0;+ // vector loop is unrolled 4x (4 accumulators in parallel)+ // we don't know how many the cpu can do at once, some can do 2, some 4+ FloatVector acc1 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector acc2 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector acc3 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector acc4 = FloatVector.zero(FLOAT_SPECIES);+ int unrolledLimit = limit - 3 * FLOAT_SPECIES.length();+ for (; i < unrolledLimit; i += 4 * FLOAT_SPECIES.length()) {+ // one+ FloatVector va = FloatVector.fromArray(FLOAT_SPECIES, a, i);+ FloatVector vb = FloatVector.fromArray(FLOAT_SPECIES, b, i);+ acc1 = acc1.add(va.mul(vb));++ // two+ FloatVector vc = FloatVector.fromArray(FLOAT_SPECIES, a, i + FLOAT_SPECIES.length());+ FloatVector vd = FloatVector.fromArray(FLOAT_SPECIES, b, i + FLOAT_SPECIES.length());+ acc2 = acc2.add(vc.mul(vd));++ // three+ FloatVector ve = FloatVector.fromArray(FLOAT_SPECIES, a, i + 2 * FLOAT_SPECIES.length());+ FloatVector vf = FloatVector.fromArray(FLOAT_SPECIES, b, i + 2 * FLOAT_SPECIES.length());+ acc3 = acc3.add(ve.mul(vf));++ // four+ FloatVector vg = FloatVector.fromArray(FLOAT_SPECIES, a, i + 3 * FLOAT_SPECIES.length());+ FloatVector vh = FloatVector.fromArray(FLOAT_SPECIES, b, i + 3 * FLOAT_SPECIES.length());+ acc4 = acc4.add(vg.mul(vh));+ }+ // vector tail: less scalar computations for unaligned sizes, esp with big vector sizes+ for (; i < limit; i += FLOAT_SPECIES.length()) {+ FloatVector va = FloatVector.fromArray(FLOAT_SPECIES, a, i);+ FloatVector vb = FloatVector.fromArray(FLOAT_SPECIES, b, i);+ acc1 = acc1.add(va.mul(vb));+ }+ // reduce+ FloatVector res1 = acc1.add(acc2);+ FloatVector res2 = acc3.add(acc4);+ return res1.add(res2).reduceLanes(ADD);+ }+@Overridepublic float cosine(float[] a, float[] b) {int i = 0;float sum = 0;float norm1 = 0;float norm2 = 0;+// if the array size is large (> 2x platform vector size), its worth the overhead to vectorize- if (a.length > 2 * PREF_FLOAT_SPECIES.length()) {- // vector loop is unrolled 4x (4 accumulators in parallel)- FloatVector sum1 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector sum2 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector sum3 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector sum4 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector norm1_1 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector norm1_2 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector norm1_3 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector norm1_4 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector norm2_1 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector norm2_2 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector norm2_3 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector norm2_4 = FloatVector.zero(PREF_FLOAT_SPECIES);- int upperBound = PREF_FLOAT_SPECIES.loopBound(a.length - 3 * PREF_FLOAT_SPECIES.length());- for (; i < upperBound; i += 4 * PREF_FLOAT_SPECIES.length()) {- FloatVector va = FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i);- FloatVector vb = FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i);- sum1 = sum1.add(va.mul(vb));- norm1_1 = norm1_1.add(va.mul(va));- norm2_1 = norm2_1.add(vb.mul(vb));- FloatVector vc =- FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i + PREF_FLOAT_SPECIES.length());- FloatVector vd =- FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i + PREF_FLOAT_SPECIES.length());- sum2 = sum2.add(vc.mul(vd));- norm1_2 = norm1_2.add(vc.mul(vc));- norm2_2 = norm2_2.add(vd.mul(vd));- FloatVector ve =- FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i + 2 * PREF_FLOAT_SPECIES.length());- FloatVector vf =- FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i + 2 * PREF_FLOAT_SPECIES.length());- sum3 = sum3.add(ve.mul(vf));- norm1_3 = norm1_3.add(ve.mul(ve));- norm2_3 = norm2_3.add(vf.mul(vf));- FloatVector vg =- FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i + 3 * PREF_FLOAT_SPECIES.length());- FloatVector vh =- FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i + 3 * PREF_FLOAT_SPECIES.length());- sum4 = sum4.add(vg.mul(vh));- norm1_4 = norm1_4.add(vg.mul(vg));- norm2_4 = norm2_4.add(vh.mul(vh));- }- // vector tail: less scalar computations for unaligned sizes, esp with big vector sizes- upperBound = PREF_FLOAT_SPECIES.loopBound(a.length);- for (; i < upperBound; i += PREF_FLOAT_SPECIES.length()) {- FloatVector va = FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i);- FloatVector vb = FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i);- sum1 = sum1.add(va.mul(vb));- norm1_1 = norm1_1.add(va.mul(va));- norm2_1 = norm2_1.add(vb.mul(vb));- }- // reduce- FloatVector sumres1 = sum1.add(sum2);- FloatVector sumres2 = sum3.add(sum4);- FloatVector norm1res1 = norm1_1.add(norm1_2);- FloatVector norm1res2 = norm1_3.add(norm1_4);- FloatVector norm2res1 = norm2_1.add(norm2_2);- FloatVector norm2res2 = norm2_3.add(norm2_4);- sum += sumres1.add(sumres2).reduceLanes(VectorOperators.ADD);- norm1 += norm1res1.add(norm1res2).reduceLanes(VectorOperators.ADD);- norm2 += norm2res1.add(norm2res2).reduceLanes(VectorOperators.ADD);+ if (a.length > 2 * FLOAT_SPECIES.length()) {+ i += FLOAT_SPECIES.loopBound(a.length);+ float[] ret = cosineBody(a, b, i);+ sum += ret[0];+ norm1 += ret[1];+ norm2 += ret[2];}+ // scalar tailfor (; i < a.length; i++) {float elem1 = a[i];float elem2 = b[i];@@ -185,56 +148,87 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {return (float) (sum / Math.sqrt((double) norm1 * (double) norm2));}+ /** vectorized cosine body */+ private float[] cosineBody(float[] a, float[] b, int limit) {+ int i = 0;+ // vector loop is unrolled 4x (4 accumulators in parallel)+ // we don't know how many the cpu can do at once, some can do 2, some 4+ FloatVector sum1 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sum2 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sum3 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector sum4 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector norm1_1 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector norm1_2 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector norm1_3 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector norm1_4 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector norm2_1 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector norm2_2 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector norm2_3 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector norm2_4 = FloatVector.zero(FLOAT_SPECIES);+ int unrolledLimit = limit - 3 * FLOAT_SPECIES.length();+ for (; i < unrolledLimit; i += 4 * FLOAT_SPECIES.length()) {+ // one+ FloatVector va = FloatVector.fromArray(FLOAT_SPECIES, a, i);+ FloatVector vb = FloatVector.fromArray(FLOAT_SPECIES, b, i);+ sum1 = sum1.add(va.mul(vb));+ norm1_1 = norm1_1.add(va.mul(va));+ norm2_1 = norm2_1.add(vb.mul(vb));++ // two+ FloatVector vc = FloatVector.fromArray(FLOAT_SPECIES, a, i + FLOAT_SPECIES.length());+ FloatVector vd = FloatVector.fromArray(FLOAT_SPECIES, b, i + FLOAT_SPECIES.length());+ sum2 = sum2.add(vc.mul(vd));+ norm1_2 = norm1_2.add(vc.mul(vc));+ norm2_2 = norm2_2.add(vd.mul(vd));++ // three+ FloatVector ve = FloatVector.fromArray(FLOAT_SPECIES, a, i + 2 * FLOAT_SPECIES.length());+ FloatVector vf = FloatVector.fromArray(FLOAT_SPECIES, b, i + 2 * FLOAT_SPECIES.length());+ sum3 = sum3.add(ve.mul(vf));+ norm1_3 = norm1_3.add(ve.mul(ve));+ norm2_3 = norm2_3.add(vf.mul(vf));++ // four+ FloatVector vg = FloatVector.fromArray(FLOAT_SPECIES, a, i + 3 * FLOAT_SPECIES.length());+ FloatVector vh = FloatVector.fromArray(FLOAT_SPECIES, b, i + 3 * FLOAT_SPECIES.length());+ sum4 = sum4.add(vg.mul(vh));+ norm1_4 = norm1_4.add(vg.mul(vg));+ norm2_4 = norm2_4.add(vh.mul(vh));+ }+ // vector tail: less scalar computations for unaligned sizes, esp with big vector sizes+ for (; i < limit; i += FLOAT_SPECIES.length()) {+ FloatVector va = FloatVector.fromArray(FLOAT_SPECIES, a, i);+ FloatVector vb = FloatVector.fromArray(FLOAT_SPECIES, b, i);+ sum1 = sum1.add(va.mul(vb));+ norm1_1 = norm1_1.add(va.mul(va));+ norm2_1 = norm2_1.add(vb.mul(vb));+ }+ // reduce+ FloatVector sumres1 = sum1.add(sum2);+ FloatVector sumres2 = sum3.add(sum4);+ FloatVector norm1res1 = norm1_1.add(norm1_2);+ FloatVector norm1res2 = norm1_3.add(norm1_4);+ FloatVector norm2res1 = norm2_1.add(norm2_2);+ FloatVector norm2res2 = norm2_3.add(norm2_4);+ return new float[] {+ sumres1.add(sumres2).reduceLanes(ADD),+ norm1res1.add(norm1res2).reduceLanes(ADD),+ norm2res1.add(norm2res2).reduceLanes(ADD)+ };+ }+@Overridepublic float squareDistance(float[] a, float[] b) {int i = 0;float res = 0;+// if the array size is large (> 2x platform vector size), its worth the overhead to vectorize- if (a.length > 2 * PREF_FLOAT_SPECIES.length()) {- // vector loop is unrolled 4x (4 accumulators in parallel)- FloatVector acc1 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector acc2 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector acc3 = FloatVector.zero(PREF_FLOAT_SPECIES);- FloatVector acc4 = FloatVector.zero(PREF_FLOAT_SPECIES);- int upperBound = PREF_FLOAT_SPECIES.loopBound(a.length - 3 * PREF_FLOAT_SPECIES.length());- for (; i < upperBound; i += 4 * PREF_FLOAT_SPECIES.length()) {- FloatVector va = FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i);- FloatVector vb = FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i);- FloatVector diff1 = va.sub(vb);- acc1 = acc1.add(diff1.mul(diff1));- FloatVector vc =- FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i + PREF_FLOAT_SPECIES.length());- FloatVector vd =- FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i + PREF_FLOAT_SPECIES.length());- FloatVector diff2 = vc.sub(vd);- acc2 = acc2.add(diff2.mul(diff2));- FloatVector ve =- FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i + 2 * PREF_FLOAT_SPECIES.length());- FloatVector vf =- FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i + 2 * PREF_FLOAT_SPECIES.length());- FloatVector diff3 = ve.sub(vf);- acc3 = acc3.add(diff3.mul(diff3));- FloatVector vg =- FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i + 3 * PREF_FLOAT_SPECIES.length());- FloatVector vh =- FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i + 3 * PREF_FLOAT_SPECIES.length());- FloatVector diff4 = vg.sub(vh);- acc4 = acc4.add(diff4.mul(diff4));- }- // vector tail: less scalar computations for unaligned sizes, esp with big vector sizes- upperBound = PREF_FLOAT_SPECIES.loopBound(a.length);- for (; i < upperBound; i += PREF_FLOAT_SPECIES.length()) {- FloatVector va = FloatVector.fromArray(PREF_FLOAT_SPECIES, a, i);- FloatVector vb = FloatVector.fromArray(PREF_FLOAT_SPECIES, b, i);- FloatVector diff = va.sub(vb);- acc1 = acc1.add(diff.mul(diff));- }- // reduce- FloatVector res1 = acc1.add(acc2);- FloatVector res2 = acc3.add(acc4);- res += res1.add(res2).reduceLanes(VectorOperators.ADD);+ if (a.length > 2 * FLOAT_SPECIES.length()) {+ i += FLOAT_SPECIES.loopBound(a.length);+ res += squareDistanceBody(a, b, i);}+ // scalar tailfor (; i < a.length; i++) {float diff = a[i] - b[i];res += diff * diff;@@ -242,138 +236,158 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {return res;}+ /** vectorized square distance body */+ private float squareDistanceBody(float[] a, float[] b, int limit) {+ int i = 0;+ // vector loop is unrolled 4x (4 accumulators in parallel)+ // we don't know how many the cpu can do at once, some can do 2, some 4+ FloatVector acc1 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector acc2 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector acc3 = FloatVector.zero(FLOAT_SPECIES);+ FloatVector acc4 = FloatVector.zero(FLOAT_SPECIES);+ int unrolledLimit = limit - 3 * FLOAT_SPECIES.length();+ for (; i < unrolledLimit; i += 4 * FLOAT_SPECIES.length()) {+ // one+ FloatVector va = FloatVector.fromArray(FLOAT_SPECIES, a, i);+ FloatVector vb = FloatVector.fromArray(FLOAT_SPECIES, b, i);+ FloatVector diff1 = va.sub(vb);+ acc1 = acc1.add(diff1.mul(diff1));++ // two+ FloatVector vc = FloatVector.fromArray(FLOAT_SPECIES, a, i + FLOAT_SPECIES.length());+ FloatVector vd = FloatVector.fromArray(FLOAT_SPECIES, b, i + FLOAT_SPECIES.length());+ FloatVector diff2 = vc.sub(vd);+ acc2 = acc2.add(diff2.mul(diff2));++ // three+ FloatVector ve = FloatVector.fromArray(FLOAT_SPECIES, a, i + 2 * FLOAT_SPECIES.length());+ FloatVector vf = FloatVector.fromArray(FLOAT_SPECIES, b, i + 2 * FLOAT_SPECIES.length());+ FloatVector diff3 = ve.sub(vf);+ acc3 = acc3.add(diff3.mul(diff3));++ // four+ FloatVector vg = FloatVector.fromArray(FLOAT_SPECIES, a, i + 3 * FLOAT_SPECIES.length());+ FloatVector vh = FloatVector.fromArray(FLOAT_SPECIES, b, i + 3 * FLOAT_SPECIES.length());+ FloatVector diff4 = vg.sub(vh);+ acc4 = acc4.add(diff4.mul(diff4));+ }+ // vector tail: less scalar computations for unaligned sizes, esp with big vector sizes+ for (; i < limit; i += FLOAT_SPECIES.length()) {+ FloatVector va = FloatVector.fromArray(FLOAT_SPECIES, a, i);+ FloatVector vb = FloatVector.fromArray(FLOAT_SPECIES, b, i);+ FloatVector diff = va.sub(vb);+ acc1 = acc1.add(diff.mul(diff));+ }+ // reduce+ FloatVector res1 = acc1.add(acc2);+ FloatVector res2 = acc3.add(acc4);+ return res1.add(res2).reduceLanes(ADD);+ }+// Binary functions, these all follow a general pattern like this://// short intermediate = a * b;- // int accumulator = accumulator + intermediate;+ // int accumulator = (int)accumulator + (int)intermediate;//// 256 or 512 bit vectors can process 64 or 128 bits at a time, respectively// intermediate results use 128 or 256 bit vectors, respectively// final accumulator uses 256 or 512 bit vectors, respectively//- // We also support 128 bit vectors, using two 128 bit accumulators.+ // We also support 128 bit vectors, going 32 bits at a time.// This is slower but still faster than not vectorizing at all.@Overridepublic int dotProduct(byte[] a, byte[] b) {int i = 0;int res = 0;+// only vectorize if we'll at least enter the loop a single time, and we have at least 128-bit// vectors (256-bit on intel to dodge performance landmines)if (a.length >= 16 && useIntegerVectors) {// compute vectorized dot product consistent with VPDPBUSD instructionif (INT_SPECIES_PREF_BIT_SIZE >= 256) {- // optimized 256/512 bit implementation, processes 8/16 bytes at a time- int upperBound = PREF_BYTE_SPECIES.loopBound(a.length);- IntVector acc = IntVector.zero(IntVector.SPECIES_PREFERRED);- for (; i < upperBound; i += PREF_BYTE_SPECIES.length()) {- ByteVector va8 = ByteVector.fromArray(PREF_BYTE_SPECIES, a, i);- ByteVector vb8 = ByteVector.fromArray(PREF_BYTE_SPECIES, b, i);- Vector<Short> va16 = va8.convertShape(VectorOperators.B2S, PREF_SHORT_SPECIES, 0);- Vector<Short> vb16 = vb8.convertShape(VectorOperators.B2S, PREF_SHORT_SPECIES, 0);- Vector<Short> prod16 = va16.mul(vb16);- Vector<Integer> prod32 =- prod16.convertShape(VectorOperators.S2I, IntVector.SPECIES_PREFERRED, 0);- acc = acc.add(prod32);- }- // reduce- res += acc.reduceLanes(VectorOperators.ADD);+ i += BYTE_SPECIES.loopBound(a.length);+ res += dotProductBody256(a, b, i);} else {- // 128-bit impl, which is tricky since we don't have SPECIES_32, it does "overlapping read"- int upperBound = ByteVector.SPECIES_64.loopBound(a.length - ByteVector.SPECIES_64.length());- IntVector acc = IntVector.zero(IntVector.SPECIES_128);- // 4 bytes at a time- for (; i < upperBound; i += ByteVector.SPECIES_64.length() >> 1) {- // load 8 bytes- ByteVector va8 = ByteVector.fromArray(ByteVector.SPECIES_64, a, i);- ByteVector vb8 = ByteVector.fromArray(ByteVector.SPECIES_64, b, i);-- // process first "half" only- Vector<Short> va16 = va8.convert(VectorOperators.B2S, 0);- Vector<Short> vb16 = vb8.convert(VectorOperators.B2S, 0);- Vector<Short> prod16 = va16.mul(vb16);-- acc = acc.add(prod16.convertShape(VectorOperators.S2I, IntVector.SPECIES_128, 0));- }- // reduce- res += acc.reduceLanes(VectorOperators.ADD);+ // tricky: we don't have SPECIES_32, so we workaround with "overlapping read"+ i += ByteVector.SPECIES_64.loopBound(a.length - ByteVector.SPECIES_64.length());+ res += dotProductBody128(a, b, i);}}+ // scalar tailfor (; i < a.length; i++) {res += b[i] * a[i];}return res;}+ /** vectorized dot product body (256+ bit vectors) */+ private int dotProductBody256(byte[] a, byte[] b, int limit) {+ IntVector acc = IntVector.zero(IntVector.SPECIES_PREFERRED);+ for (int i = 0; i < limit; i += BYTE_SPECIES.length()) {+ ByteVector va8 = ByteVector.fromArray(BYTE_SPECIES, a, i);+ ByteVector vb8 = ByteVector.fromArray(BYTE_SPECIES, b, i);++ // 16-bit multiply+ Vector<Short> va16 = va8.convertShape(B2S, SHORT_SPECIES, 0);+ Vector<Short> vb16 = vb8.convertShape(B2S, SHORT_SPECIES, 0);+ Vector<Short> prod16 = va16.mul(vb16);++ // 32-bit add+ Vector<Integer> prod32 = prod16.convertShape(S2I, IntVector.SPECIES_PREFERRED, 0);+ acc = acc.add(prod32);+ }+ // reduce+ return acc.reduceLanes(ADD);+ }++ /** vectorized dot product body (128 bit vectors) */+ private int dotProductBody128(byte[] a, byte[] b, int limit) {+ IntVector acc = IntVector.zero(IntVector.SPECIES_128);+ // 4 bytes at a time (re-loading half the vector each time!)+ for (int i = 0; i < limit; i += ByteVector.SPECIES_64.length() >> 1) {+ // load 8 bytes+ ByteVector va8 = ByteVector.fromArray(ByteVector.SPECIES_64, a, i);+ ByteVector vb8 = ByteVector.fromArray(ByteVector.SPECIES_64, b, i);++ // process first "half" only: 16-bit multiply+ Vector<Short> va16 = va8.convert(B2S, 0);+ Vector<Short> vb16 = vb8.convert(B2S, 0);+ Vector<Short> prod16 = va16.mul(vb16);++ // 32-bit add+ acc = acc.add(prod16.convertShape(S2I, IntVector.SPECIES_128, 0));+ }+ // reduce+ return acc.reduceLanes(ADD);+ }+@Overridepublic float cosine(byte[] a, byte[] b) {int i = 0;int sum = 0;int norm1 = 0;int norm2 = 0;+// only vectorize if we'll at least enter the loop a single time, and we have at least 128-bit// vectors (256-bit on intel to dodge performance landmines)if (a.length >= 16 && useIntegerVectors) {+ final float[] ret;if (INT_SPECIES_PREF_BIT_SIZE >= 256) {- // optimized 256/512 bit implementation, processes 8/16 bytes at a time- int upperBound = PREF_BYTE_SPECIES.loopBound(a.length);- IntVector accSum = IntVector.zero(IntVector.SPECIES_PREFERRED);- IntVector accNorm1 = IntVector.zero(IntVector.SPECIES_PREFERRED);- IntVector accNorm2 = IntVector.zero(IntVector.SPECIES_PREFERRED);- for (; i < upperBound; i += PREF_BYTE_SPECIES.length()) {- ByteVector va8 = ByteVector.fromArray(PREF_BYTE_SPECIES, a, i);- ByteVector vb8 = ByteVector.fromArray(PREF_BYTE_SPECIES, b, i);- Vector<Short> va16 = va8.convertShape(VectorOperators.B2S, PREF_SHORT_SPECIES, 0);- Vector<Short> vb16 = vb8.convertShape(VectorOperators.B2S, PREF_SHORT_SPECIES, 0);- Vector<Short> prod16 = va16.mul(vb16);- Vector<Short> norm1_16 = va16.mul(va16);- Vector<Short> norm2_16 = vb16.mul(vb16);- Vector<Integer> prod32 =- prod16.convertShape(VectorOperators.S2I, IntVector.SPECIES_PREFERRED, 0);- Vector<Integer> norm1_32 =- norm1_16.convertShape(VectorOperators.S2I, IntVector.SPECIES_PREFERRED, 0);- Vector<Integer> norm2_32 =- norm2_16.convertShape(VectorOperators.S2I, IntVector.SPECIES_PREFERRED, 0);- accSum = accSum.add(prod32);- accNorm1 = accNorm1.add(norm1_32);- accNorm2 = accNorm2.add(norm2_32);- }- // reduce- sum += accSum.reduceLanes(VectorOperators.ADD);- norm1 += accNorm1.reduceLanes(VectorOperators.ADD);- norm2 += accNorm2.reduceLanes(VectorOperators.ADD);+ i += BYTE_SPECIES.loopBound(a.length);+ ret = cosineBody256(a, b, i);} else {- // 128-bit impl, which is tricky since we don't have SPECIES_32, it does "overlapping read"- int upperBound = ByteVector.SPECIES_64.loopBound(a.length - ByteVector.SPECIES_64.length());- IntVector accSum = IntVector.zero(IntVector.SPECIES_128);- IntVector accNorm1 = IntVector.zero(IntVector.SPECIES_128);- IntVector accNorm2 = IntVector.zero(IntVector.SPECIES_128);- for (; i < upperBound; i += ByteVector.SPECIES_64.length() >> 1) {- ByteVector va8 = ByteVector.fromArray(ByteVector.SPECIES_64, a, i);- ByteVector vb8 = ByteVector.fromArray(ByteVector.SPECIES_64, b, i);-- // process first half only- Vector<Short> va16 = va8.convert(VectorOperators.B2S, 0);- Vector<Short> vb16 = vb8.convert(VectorOperators.B2S, 0);- Vector<Short> norm1_16 = va16.mul(va16);- Vector<Short> norm2_16 = vb16.mul(vb16);- Vector<Short> prod16 = va16.mul(vb16);-- // sum into accumulators- accNorm1 =- accNorm1.add(norm1_16.convertShape(VectorOperators.S2I, IntVector.SPECIES_128, 0));- accNorm2 =- accNorm2.add(norm2_16.convertShape(VectorOperators.S2I, IntVector.SPECIES_128, 0));- accSum = accSum.add(prod16.convertShape(VectorOperators.S2I, IntVector.SPECIES_128, 0));- }- // reduce- sum += accSum.reduceLanes(VectorOperators.ADD);- norm1 += accNorm1.reduceLanes(VectorOperators.ADD);- norm2 += accNorm2.reduceLanes(VectorOperators.ADD);+ // tricky: we don't have SPECIES_32, so we workaround with "overlapping read"+ i += ByteVector.SPECIES_64.loopBound(a.length - ByteVector.SPECIES_64.length());+ ret = cosineBody128(a, b, i);}+ sum += ret[0];+ norm1 += ret[1];+ norm2 += ret[2];}+ // scalar tailfor (; i < a.length; i++) {byte elem1 = a[i];byte elem2 = b[i];@@ -384,58 +398,131 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {return (float) (sum / Math.sqrt((double) norm1 * (double) norm2));}+ /** vectorized cosine body (256+ bit vectors) */+ private float[] cosineBody256(byte[] a, byte[] b, int limit) {+ // optimized 256/512 bit implementation, processes 8/16 bytes at a time+ IntVector accSum = IntVector.zero(IntVector.SPECIES_PREFERRED);+ IntVector accNorm1 = IntVector.zero(IntVector.SPECIES_PREFERRED);+ IntVector accNorm2 = IntVector.zero(IntVector.SPECIES_PREFERRED);+ for (int i = 0; i < limit; i += BYTE_SPECIES.length()) {+ ByteVector va8 = ByteVector.fromArray(BYTE_SPECIES, a, i);+ ByteVector vb8 = ByteVector.fromArray(BYTE_SPECIES, b, i);++ // 16-bit multiply+ Vector<Short> va16 = va8.convertShape(B2S, SHORT_SPECIES, 0);+ Vector<Short> vb16 = vb8.convertShape(B2S, SHORT_SPECIES, 0);+ Vector<Short> prod16 = va16.mul(vb16);+ Vector<Short> norm1_16 = va16.mul(va16);+ Vector<Short> norm2_16 = vb16.mul(vb16);++ // sum into accumulators: 32-bit add+ Vector<Integer> prod32 = prod16.convertShape(S2I, IntVector.SPECIES_PREFERRED, 0);+ Vector<Integer> norm1_32 = norm1_16.convertShape(S2I, IntVector.SPECIES_PREFERRED, 0);+ Vector<Integer> norm2_32 = norm2_16.convertShape(S2I, IntVector.SPECIES_PREFERRED, 0);+ accSum = accSum.add(prod32);+ accNorm1 = accNorm1.add(norm1_32);+ accNorm2 = accNorm2.add(norm2_32);+ }+ // reduce+ return new float[] {+ accSum.reduceLanes(ADD), accNorm1.reduceLanes(ADD), accNorm2.reduceLanes(ADD)+ };+ }++ /** vectorized cosine body (128 bit vectors) */+ private float[] cosineBody128(byte[] a, byte[] b, int limit) {+ IntVector accSum = IntVector.zero(IntVector.SPECIES_128);+ IntVector accNorm1 = IntVector.zero(IntVector.SPECIES_128);+ IntVector accNorm2 = IntVector.zero(IntVector.SPECIES_128);+ for (int i = 0; i < limit; i += ByteVector.SPECIES_64.length() >> 1) {+ ByteVector va8 = ByteVector.fromArray(ByteVector.SPECIES_64, a, i);+ ByteVector vb8 = ByteVector.fromArray(ByteVector.SPECIES_64, b, i);++ // process first half only: 16-bit multiply+ Vector<Short> va16 = va8.convert(B2S, 0);+ Vector<Short> vb16 = vb8.convert(B2S, 0);+ Vector<Short> norm1_16 = va16.mul(va16);+ Vector<Short> norm2_16 = vb16.mul(vb16);+ Vector<Short> prod16 = va16.mul(vb16);++ // sum into accumulators: 32-bit add+ accNorm1 = accNorm1.add(norm1_16.convertShape(S2I, IntVector.SPECIES_128, 0));+ accNorm2 = accNorm2.add(norm2_16.convertShape(S2I, IntVector.SPECIES_128, 0));+ accSum = accSum.add(prod16.convertShape(S2I, IntVector.SPECIES_128, 0));+ }+ // reduce+ return new float[] {+ accSum.reduceLanes(ADD), accNorm1.reduceLanes(ADD), accNorm2.reduceLanes(ADD)+ };+ }+@Overridepublic int squareDistance(byte[] a, byte[] b) {int i = 0;int res = 0;+// only vectorize if we'll at least enter the loop a single time, and we have at least 128-bit// vectors (256-bit on intel to dodge performance landmines)if (a.length >= 16 && useIntegerVectors) {if (INT_SPECIES_PREF_BIT_SIZE >= 256) {- // optimized 256/512 bit implementation, processes 8/16 bytes at a time- int upperBound = PREF_BYTE_SPECIES.loopBound(a.length);- IntVector acc = IntVector.zero(IntVector.SPECIES_PREFERRED);- for (; i < upperBound; i += PREF_BYTE_SPECIES.length()) {- ByteVector va8 = ByteVector.fromArray(PREF_BYTE_SPECIES, a, i);- ByteVector vb8 = ByteVector.fromArray(PREF_BYTE_SPECIES, b, i);- Vector<Short> va16 = va8.convertShape(VectorOperators.B2S, PREF_SHORT_SPECIES, 0);- Vector<Short> vb16 = vb8.convertShape(VectorOperators.B2S, PREF_SHORT_SPECIES, 0);- Vector<Short> diff16 = va16.sub(vb16);- Vector<Integer> diff32 =- diff16.convertShape(VectorOperators.S2I, IntVector.SPECIES_PREFERRED, 0);- acc = acc.add(diff32.mul(diff32));- }- // reduce- res += acc.reduceLanes(VectorOperators.ADD);+ i += BYTE_SPECIES.loopBound(a.length);+ res += squareDistanceBody256(a, b, i);} else {- // 128-bit implementation, which must "split up" vectors due to widening conversions- int upperBound = ByteVector.SPECIES_64.loopBound(a.length);- IntVector acc1 = IntVector.zero(IntVector.SPECIES_128);- IntVector acc2 = IntVector.zero(IntVector.SPECIES_128);- for (; i < upperBound; i += ByteVector.SPECIES_64.length()) {- ByteVector va8 = ByteVector.fromArray(ByteVector.SPECIES_64, a, i);- ByteVector vb8 = ByteVector.fromArray(ByteVector.SPECIES_64, b, i);- // expand each byte vector into short vector and subtract- Vector<Short> va16 = va8.convertShape(VectorOperators.B2S, ShortVector.SPECIES_128, 0);- Vector<Short> vb16 = vb8.convertShape(VectorOperators.B2S, ShortVector.SPECIES_128, 0);- Vector<Short> diff16 = va16.sub(vb16);- // split each short vector into two int vectors, square, and add- Vector<Integer> diff32_1 =- diff16.convertShape(VectorOperators.S2I, IntVector.SPECIES_128, 0);- Vector<Integer> diff32_2 =- diff16.convertShape(VectorOperators.S2I, IntVector.SPECIES_128, 1);- acc1 = acc1.add(diff32_1.mul(diff32_1));- acc2 = acc2.add(diff32_2.mul(diff32_2));- }- // reduce- res += acc1.add(acc2).reduceLanes(VectorOperators.ADD);+ i += ByteVector.SPECIES_64.loopBound(a.length);+ res += squareDistanceBody128(a, b, i);}}+ // scalar tailfor (; i < a.length; i++) {int diff = a[i] - b[i];res += diff * diff;}return res;}++ /** vectorized square distance body (256+ bit vectors) */+ private int squareDistanceBody256(byte[] a, byte[] b, int limit) {+ IntVector acc = IntVector.zero(IntVector.SPECIES_PREFERRED);+ for (int i = 0; i < limit; i += BYTE_SPECIES.length()) {+ ByteVector va8 = ByteVector.fromArray(BYTE_SPECIES, a, i);+ ByteVector vb8 = ByteVector.fromArray(BYTE_SPECIES, b, i);++ // 16-bit sub+ Vector<Short> va16 = va8.convertShape(B2S, SHORT_SPECIES, 0);+ Vector<Short> vb16 = vb8.convertShape(B2S, SHORT_SPECIES, 0);+ Vector<Short> diff16 = va16.sub(vb16);++ // 32-bit multiply and add into accumulators+ Vector<Integer> diff32 = diff16.convertShape(S2I, IntVector.SPECIES_PREFERRED, 0);+ acc = acc.add(diff32.mul(diff32));+ }+ // reduce+ return acc.reduceLanes(ADD);+ }++ /** vectorized square distance body (128 bit vectors) */+ private int squareDistanceBody128(byte[] a, byte[] b, int limit) {+ // 128-bit implementation, which must "split up" vectors due to widening conversions+ // it doesn't help to do the overlapping read trick, due to 32-bit multiply in the formula+ IntVector acc1 = IntVector.zero(IntVector.SPECIES_128);+ IntVector acc2 = IntVector.zero(IntVector.SPECIES_128);+ for (int i = 0; i < limit; i += ByteVector.SPECIES_64.length()) {+ ByteVector va8 = ByteVector.fromArray(ByteVector.SPECIES_64, a, i);+ ByteVector vb8 = ByteVector.fromArray(ByteVector.SPECIES_64, b, i);++ // 16-bit sub+ Vector<Short> va16 = va8.convertShape(B2S, ShortVector.SPECIES_128, 0);+ Vector<Short> vb16 = vb8.convertShape(B2S, ShortVector.SPECIES_128, 0);+ Vector<Short> diff16 = va16.sub(vb16);++ // 32-bit multiply and add into accumulators+ Vector<Integer> diff32_1 = diff16.convertShape(S2I, IntVector.SPECIES_128, 0);+ Vector<Integer> diff32_2 = diff16.convertShape(S2I, IntVector.SPECIES_128, 1);+ acc1 = acc1.add(diff32_1.mul(diff32_1));+ acc2 = acc2.add(diff32_2.mul(diff32_2));+ }+ // reduce+ return acc1.add(acc2).reduceLanes(ADD);+ }}