Apache Lucene open-source search software
Fix silly inefficiencies in HnswGraph.NodesIterator implementation (#15425)
Michael Sokolov08538bbaf0e7
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex c53632a7c2e..590f0def577 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -187,6 +187,8 @@ Improvements* GITHUB#15332: Add PhraseQuery.Builder.setMaxTerms() method to limit the maximum number of terms and excessive memory use (linyunanit)+* GITHUB#15425: Refactoring internal HnswGraph.NodesIterator to avoid unneeded copying and sorting (Mike Sokolov)+Optimizations---------------------* GITHUB#15140: Optimize TopScoreDocCollector with TernaryLongHeap for improved performance over Binary-LongHeap. (Ramakrishna Chilaka)diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.javaindex 9bb320e7128..2c99e782f2e 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java@@ -575,9 +575,9 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {@Overridepublic NodesIterator getNodesOnLevel(int level) {if (level == 0) {- return new ArrayNodesIterator(size());+ return new DenseNodesIterator(size());} else {- return new ArrayNodesIterator(nodesByLevel[level], nodesByLevel[level].length);+ return new ArrayNodesIterator(nodesByLevel[level]);}}}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91OnHeapHnswGraph.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91OnHeapHnswGraph.javaindex 7effb2dd053..9724599a495 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91OnHeapHnswGraph.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91OnHeapHnswGraph.java@@ -173,7 +173,7 @@ public final class Lucene91OnHeapHnswGraph extends HnswGraph {@Overridepublic NodesIterator getNodesOnLevel(int level) {if (level == 0) {- return new ArrayNodesIterator(size());+ return new DenseNodesIterator(size());} else {return new ArrayNodesIterator(nodesByLevel.get(level), graph.get(level).size());}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.javaindex 32dab1b6c7c..a749e74c7fb 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java@@ -485,9 +485,9 @@ public final class Lucene92HnswVectorsReader extends KnnVectorsReader {@Overridepublic NodesIterator getNodesOnLevel(int level) {if (level == 0) {- return new ArrayNodesIterator(size());+ return new DenseNodesIterator(size());} else {- return new ArrayNodesIterator(nodesByLevel[level], nodesByLevel[level].length);+ return new ArrayNodesIterator(nodesByLevel[level]);}}}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.javaindex 5b937375f1a..3bacd40482a 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java@@ -537,9 +537,9 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {@Overridepublic NodesIterator getNodesOnLevel(int level) {if (level == 0) {- return new ArrayNodesIterator(size());+ return new DenseNodesIterator(size());} else {- return new ArrayNodesIterator(nodesByLevel[level], nodesByLevel[level].length);+ return new ArrayNodesIterator(nodesByLevel[level]);}}}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.javaindex 7fa6ffbaeda..20571783ab6 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java@@ -566,9 +566,9 @@ public final class Lucene95HnswVectorsReader extends KnnVectorsReader implements@Overridepublic NodesIterator getNodesOnLevel(int level) {if (level == 0) {- return new ArrayNodesIterator(size());+ return new DenseNodesIterator(size());} else {- return new ArrayNodesIterator(nodesByLevel[level], nodesByLevel[level].length);+ return new ArrayNodesIterator(nodesByLevel[level]);}}}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsWriter.javaindex a984a3ef1f8..af78293e35a 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsWriter.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsWriter.java@@ -230,11 +230,11 @@ public final class Lucene91HnswVectorsWriter extends BufferingKnnVectorsWriter {} else {meta.writeInt(graph.numLevels());for (int level = 0; level < graph.numLevels(); level++) {- int[] sortedNodes = HnswGraph.NodesIterator.getSortedNodes(graph.getNodesOnLevel(level));- meta.writeInt(sortedNodes.length); // number of nodes on a level+ HnswGraph.NodesIterator sortedNodes = graph.getSortedNodes(level);+ meta.writeInt(sortedNodes.size()); // number of nodes on a levelif (level > 0) {- for (int node : sortedNodes) {- meta.writeInt(node); // list of nodes on a level+ while (sortedNodes.hasNext()) {+ meta.writeInt(sortedNodes.next()); // list of nodes on a level}}}@@ -259,9 +259,9 @@ public final class Lucene91HnswVectorsWriter extends BufferingKnnVectorsWriter {// write vectors' neighbours on each level into the vectorIndex fileint countOnLevel0 = graph.size();for (int level = 0; level < graph.numLevels(); level++) {- int[] sortedNodes = HnswGraph.NodesIterator.getSortedNodes(graph.getNodesOnLevel(level));- for (int node : sortedNodes) {- Lucene91NeighborArray neighbors = graph.getNeighbors(level, node);+ HnswGraph.NodesIterator sortedNodes = graph.getSortedNodes(level);+ while (sortedNodes.hasNext()) {+ Lucene91NeighborArray neighbors = graph.getNeighbors(level, sortedNodes.next());int size = neighbors.size();vectorIndex.writeInt(size);// Destructively modify; it's ok we are discarding it after thisdiff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsWriter.javaindex bf1c89a536d..917e5dd583f 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsWriter.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsWriter.java@@ -267,11 +267,11 @@ public final class Lucene92HnswVectorsWriter extends BufferingKnnVectorsWriter {} else {meta.writeInt(graph.numLevels());for (int level = 0; level < graph.numLevels(); level++) {- int[] sortedNodes = HnswGraph.NodesIterator.getSortedNodes(graph.getNodesOnLevel(level));- meta.writeInt(sortedNodes.length); // number of nodes on a level+ HnswGraph.NodesIterator sortedNodes = graph.getSortedNodes(level);+ meta.writeInt(sortedNodes.size()); // number of nodes on a levelif (level > 0) {- for (int node : sortedNodes) {- meta.writeInt(node); // list of nodes on a level+ while (sortedNodes.hasNext()) {+ meta.writeInt(sortedNodes.next()); // list of nodes on a level}}}@@ -295,9 +295,9 @@ public final class Lucene92HnswVectorsWriter extends BufferingKnnVectorsWriter {int countOnLevel0 = graph.size();for (int level = 0; level < graph.numLevels(); level++) {int maxConnOnLevel = level == 0 ? (M * 2) : M;- int[] sortedNodes = HnswGraph.NodesIterator.getSortedNodes(graph.getNodesOnLevel(level));- for (int node : sortedNodes) {- NeighborArray neighbors = graph.getNeighbors(level, node);+ HnswGraph.NodesIterator sortedNodes = graph.getSortedNodes(level);+ while (sortedNodes.hasNext()) {+ NeighborArray neighbors = graph.getNeighbors(level, sortedNodes.next());int size = neighbors.size();vectorIndex.writeInt(size);// Destructively modify; it's ok we are discarding it after thisdiff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.javaindex fdc5d5da146..da62db66c89 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java@@ -354,7 +354,7 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {if (level == 0) {return graph.getNodesOnLevel(0);} else {- return new ArrayNodesIterator(nodesByLevel.get(level), nodesByLevel.get(level).length);+ return new ArrayNodesIterator(nodesByLevel.get(level));}}};@@ -492,9 +492,9 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {int countOnLevel0 = graph.size();for (int level = 0; level < graph.numLevels(); level++) {int maxConnOnLevel = level == 0 ? (M * 2) : M;- int[] sortedNodes = HnswGraph.NodesIterator.getSortedNodes(graph.getNodesOnLevel(level));- for (int node : sortedNodes) {- NeighborArray neighbors = graph.getNeighbors(level, node);+ HnswGraph.NodesIterator sortedNodes = graph.getSortedNodes(level);+ while (sortedNodes.hasNext()) {+ NeighborArray neighbors = graph.getNeighbors(level, sortedNodes.next());int size = neighbors.size();vectorIndex.writeInt(size);// Destructively modify; it's ok we are discarding it after this@@ -578,11 +578,11 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {} else {meta.writeInt(graph.numLevels());for (int level = 0; level < graph.numLevels(); level++) {- int[] sortedNodes = HnswGraph.NodesIterator.getSortedNodes(graph.getNodesOnLevel(level));- meta.writeInt(sortedNodes.length); // number of nodes on a level+ HnswGraph.NodesIterator sortedNodes = graph.getSortedNodes(level);+ meta.writeInt(sortedNodes.size()); // number of nodes on a levelif (level > 0) {- for (int node : sortedNodes) {- meta.writeInt(node); // list of nodes on a level+ while (sortedNodes.hasNext()) {+ meta.writeInt(sortedNodes.next()); // list of nodes on a level}}}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.javaindex 943e21c560d..75abbb3e60b 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java@@ -381,7 +381,7 @@ public final class Lucene95HnswVectorsWriter extends KnnVectorsWriter {if (level == 0) {return graph.getNodesOnLevel(0);} else {- return new ArrayNodesIterator(nodesByLevel.get(level), nodesByLevel.get(level).length);+ return new ArrayNodesIterator(nodesByLevel.get(level));}}};@@ -539,11 +539,11 @@ public final class Lucene95HnswVectorsWriter extends KnnVectorsWriter {int countOnLevel0 = graph.size();int[][] offsets = new int[graph.numLevels()][];for (int level = 0; level < graph.numLevels(); level++) {- int[] sortedNodes = HnswGraph.NodesIterator.getSortedNodes(graph.getNodesOnLevel(level));- offsets[level] = new int[sortedNodes.length];+ HnswGraph.NodesIterator sortedNodes = graph.getSortedNodes(level);+ offsets[level] = new int[sortedNodes.size()];int nodeOffsetId = 0;- for (int node : sortedNodes) {- NeighborArray neighbors = graph.getNeighbors(level, node);+ while (sortedNodes.hasNext()) {+ NeighborArray neighbors = graph.getNeighbors(level, sortedNodes.next());int size = neighbors.size();// Write size in VInt as the neighbors list is typically smalllong offsetStart = vectorIndex.getFilePointer();diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.javaindex 001fb9982d3..d7d95d45486 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java@@ -615,9 +615,9 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader@Overridepublic NodesIterator getNodesOnLevel(int level) {if (level == 0) {- return new ArrayNodesIterator(size());+ return new DenseNodesIterator(size());} else {- return new ArrayNodesIterator(nodesByLevel[level], nodesByLevel[level].length);+ return new ArrayNodesIterator(nodesByLevel[level]);}}}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.javaindex 89f23bb5b0f..d2526cff3ab 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java@@ -360,7 +360,7 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {if (level == 0) {return graph.getNodesOnLevel(0);} else {- return new ArrayNodesIterator(nodesByLevel.get(level), nodesByLevel.get(level).length);+ return new ArrayNodesIterator(nodesByLevel.get(level));}}};@@ -409,7 +409,6 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {// build the graph using the temporary vector data// we use Lucene99HnswVectorsReader.DenseOffHeapVectorValues for the graph construction// doesn't need to know docIds- // TODO: separate random access vector values from DocIdSetIterator?OnHeapHnswGraph graph = null;int[][] vectorIndexNodeOffsets = null;// Check if we should bypass graph building for tiny segments@@ -475,11 +474,11 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {int[][] offsets = new int[graph.numLevels()][];int[] scratch = new int[graph.maxConn() * 2];for (int level = 0; level < graph.numLevels(); level++) {- int[] sortedNodes = NodesIterator.getSortedNodes(graph.getNodesOnLevel(level));- offsets[level] = new int[sortedNodes.length];+ NodesIterator sortedNodes = graph.getSortedNodes(level);+ offsets[level] = new int[sortedNodes.size()];int nodeOffsetId = 0;- for (int node : sortedNodes) {- NeighborArray neighbors = graph.getNeighbors(level, node);+ while (sortedNodes.hasNext()) {+ NeighborArray neighbors = graph.getNeighbors(level, sortedNodes.next());int size = neighbors.size();// Write size in VInt as the neighbors list is typically smalllong offsetStart = vectorIndex.getFilePointer();diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraph.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraph.javaindex 264fdfeb675..3676b7fd09b 100644--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraph.java+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraph.java@@ -141,10 +141,23 @@ public abstract class HnswGraph {@Overridepublic NodesIterator getNodesOnLevel(int level) {- return ArrayNodesIterator.EMPTY;+ return DenseNodesIterator.EMPTY;}};+ public NodesIterator getSortedNodes(int level) throws IOException {+ if (level == 0) {+ return new DenseNodesIterator(size());+ }+ NodesIterator nodesOnLevel = getNodesOnLevel(level);+ int[] sortedNodes = new int[nodesOnLevel.size()];+ for (int n = 0; nodesOnLevel.hasNext(); n++) {+ sortedNodes[n] = nodesOnLevel.nextInt();+ }+ Arrays.sort(sortedNodes);+ return new ArrayNodesIterator(sortedNodes);+ }+/*** Iterator over the graph nodes on a certain level. Iterator also provides the size – the total* number of nodes to be iterated over. The nodes are NOT guaranteed to be presented in any@@ -170,36 +183,59 @@ public abstract class HnswGraph {* @return The number of integers written to `dest`*/public abstract int consume(int[] dest);-- public static int[] getSortedNodes(NodesIterator nodesOnLevel) {- int[] sortedNodes = new int[nodesOnLevel.size()];- for (int n = 0; nodesOnLevel.hasNext(); n++) {- sortedNodes[n] = nodesOnLevel.nextInt();- }- Arrays.sort(sortedNodes);- return sortedNodes;- }}/** NodesIterator that accepts nodes as an integer array. */public static class ArrayNodesIterator extends NodesIterator {- private static final NodesIterator EMPTY = new ArrayNodesIterator(0);private final int[] nodes;private int cur = 0;- /** Constructor for iterator based on integer array representing nodes */+ /** Normal constructor */+ public ArrayNodesIterator(int[] nodes) {+ this(nodes, nodes.length);+ }++ /** Constructor that allows overriding size, used only for back-compat */public ArrayNodesIterator(int[] nodes, int size) {super(size);- assert nodes != null;- assert size <= nodes.length;this.nodes = nodes;}- /** Constructor for iterator based on the size */- public ArrayNodesIterator(int size) {+ @Override+ public int consume(int[] dest) {+ if (hasNext() == false) {+ throw new NoSuchElementException();+ }+ int numToCopy = Math.min(size - cur, dest.length);+ System.arraycopy(nodes, cur, dest, 0, numToCopy);+ cur += numToCopy;+ return numToCopy;+ }++ @Override+ public int nextInt() {+ if (hasNext() == false) {+ throw new NoSuchElementException();+ }+ return nodes[cur++];+ }++ @Override+ public boolean hasNext() {+ return cur < size;+ }+ }++ /** NodesIterator that enumerates [0, size) */+ public static class DenseNodesIterator extends NodesIterator {+ private static final NodesIterator EMPTY = new DenseNodesIterator(0);++ private int cur = 0;++ /** Sole constructor */+ public DenseNodesIterator(int size) {super(size);- this.nodes = null;}@Override@@ -208,14 +244,9 @@ public abstract class HnswGraph {throw new NoSuchElementException();}int numToCopy = Math.min(size - cur, dest.length);- if (nodes == null) {- for (int i = 0; i < numToCopy; i++) {- dest[i] = cur + i;- }- return numToCopy;+ for (int i = 0; i < numToCopy; i++) {+ dest[i] = cur + i;}- System.arraycopy(nodes, cur, dest, 0, numToCopy);- cur += numToCopy;return numToCopy;}@@ -224,11 +255,7 @@ public abstract class HnswGraph {if (hasNext() == false) {throw new NoSuchElementException();}- if (nodes == null) {- return cur++;- } else {- return nodes[cur++];- }+ return cur++;}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswUtil.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswUtil.javaindex ab1ea556acc..3492350a3b3 100644--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswUtil.java+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswUtil.java@@ -90,7 +90,7 @@ public class HnswUtil {HnswGraph.NodesIterator entryPoints;// System.out.println("components level=" + level);if (level == hnsw.numLevels() - 1) {- entryPoints = new HnswGraph.ArrayNodesIterator(new int[] {hnsw.entryNode()}, 1);+ entryPoints = new HnswGraph.ArrayNodesIterator(new int[] {hnsw.entryNode()});} else {entryPoints = hnsw.getNodesOnLevel(level + 1);}diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.javaindex eb9014f7357..adda151df93 100644--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java@@ -276,7 +276,7 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {"graph build not complete, size=" + size() + " maxNodeId=" + maxNodeId());}if (level == 0) {- return new ArrayNodesIterator(size());+ return new DenseNodesIterator(size());} else {generateLevelToNodes();return new CollectionNodesIterator(levelToNodes[level]);