Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Introduced immutable indexes to solve the problem of dying-peers-when-growing problem:
Michael Peter Christend05db9e6bb5d
diff --git a/source/net/yacy/cora/storage/HandleMap.java b/source/net/yacy/cora/storage/HandleMap.javaindex 81562436a..8b7728758 100644--- a/source/net/yacy/cora/storage/HandleMap.java+++ b/source/net/yacy/cora/storage/HandleMap.java@@ -20,38 +20,11 @@package net.yacy.cora.storage;-import java.io.File;import java.io.IOException;-import java.util.ArrayList;-import java.util.Map;-import net.yacy.cora.order.CloneableIterator;import net.yacy.cora.util.SpaceExceededException;-public interface HandleMap extends Iterable<Map.Entry<byte[], Long>> {-- public long mem();-- public void optimize();-- /**- * write a dump of the index to a file. All entries are written in order- * which makes it possible to read them again in a fast way- * @param file- * @return the number of written entries- * @throws IOException- */- public int dump(final File file) throws IOException;-- public void clear();-- public byte[] smallestKey();-- public byte[] largestKey();-- public boolean has(final byte[] key);-- public long get(final byte[] key);+public interface HandleMap extends ImmutableHandleMap {/*** Adds the key-value pair to the index.@@ -61,30 +34,14 @@ public interface HandleMap extends Iterable<Map.Entry<byte[], Long>> {* @throws IOException* @throws SpaceExceededException*/- public long put(final byte[] key, final long l) throws SpaceExceededException;-- public void putUnique(final byte[] key, final long l) throws SpaceExceededException;-- public long add(final byte[] key, final long a) throws SpaceExceededException;-- public long inc(final byte[] key) throws SpaceExceededException;-- public long dec(final byte[] key) throws SpaceExceededException;-- public ArrayList<long[]> removeDoubles() throws SpaceExceededException;-- public ArrayList<byte[]> top(final int count);-- public long remove(final byte[] key);-- public long removeone();+ public long put(final byte[] key, final long l) throws SpaceExceededException; // mutable operation();- public int size();+ public void putUnique(final byte[] key, final long l) throws SpaceExceededException; // mutable operation();- public boolean isEmpty();+ public long add(final byte[] key, final long a) throws SpaceExceededException; // mutable operation();- public CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey);+ public long inc(final byte[] key) throws SpaceExceededException; // mutable operation();- public void close();+ public long dec(final byte[] key) throws SpaceExceededException; // mutable operation();}diff --git a/source/net/yacy/cora/storage/ImmutableHandleMap.java b/source/net/yacy/cora/storage/ImmutableHandleMap.javanew file mode 100644index 000000000..4ee9ca575--- /dev/null+++ b/source/net/yacy/cora/storage/ImmutableHandleMap.java@@ -0,0 +1,72 @@+/**+ * ImmutableHandleMap+ * Copyright 2009 by Michael Peter Christen; [email protected], Frankfurt a. M., Germany+ * First published 26.07.2012 on http://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.cora.storage;++import java.io.File;+import java.io.IOException;+import java.util.ArrayList;+import java.util.Map;++import net.yacy.cora.order.CloneableIterator;+import net.yacy.cora.util.SpaceExceededException;++public interface ImmutableHandleMap extends Iterable<Map.Entry<byte[], Long>> {++ public long mem();++ public void optimize();++ /**+ * write a dump of the index to a file. All entries are written in order+ * which makes it possible to read them again in a fast way+ * @param file+ * @return the number of written entries+ * @throws IOException+ */+ public int dump(final File file) throws IOException;++ public void clear();++ public byte[] smallestKey();++ public byte[] largestKey();++ public boolean has(final byte[] key);++ public long get(final byte[] key);++ public ArrayList<long[]> removeDoubles() throws SpaceExceededException;++ public ArrayList<byte[]> top(final int count);++ public long remove(final byte[] key);++ public long removeone();++ public int size();++ public boolean isEmpty();++ public CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey);++ public void close();++}diff --git a/source/net/yacy/cora/util/ChunkedBytes.java b/source/net/yacy/cora/util/ChunkedBytes.javaindex 1f31ca17b..95c1085e5 100644--- a/source/net/yacy/cora/util/ChunkedBytes.java+++ b/source/net/yacy/cora/util/ChunkedBytes.java@@ -25,8 +25,10 @@ import java.io.EOFException;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;+import java.io.UncheckedIOException;import java.nio.ByteBuffer;import java.nio.MappedByteBuffer;+import java.nio.ReadOnlyBufferException;import java.nio.channels.FileChannel;import java.nio.file.Files;import java.nio.file.OpenOption;@@ -34,6 +36,7 @@ import java.nio.file.Path;import java.nio.file.StandardOpenOption;import java.util.ArrayList;import java.util.List;+import java.util.concurrent.atomic.AtomicInteger;import net.yacy.cora.document.encoding.UTF8;@@ -49,10 +52,12 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objecprivate final List<Segment> segments;private long size;+ private volatile boolean closed;public ChunkedBytes() {this.segments = new ArrayList<>();this.size = 0;+ this.closed = false;}public ChunkedBytes(InputStream in) throws IOException {@@ -70,10 +75,13 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objecthis.append(UTF8.getBytes(initialData));}- private ChunkedBytes(List<Segment> segments, long size) {- this.segments = new ArrayList<>(segments);- this.size = size;- }+ private ChunkedBytes(final List<Segment> segments, final long size,+ final List<FileBacking> fileBackings) {+ this.segments = new ArrayList<>(segments);+ this.size = size;+ this.fileBackings.addAll(fileBackings);+ this.closed = false;+ }/** Represents one contiguous region in the logical address space. */private static final class Segment implements Closeable, Cloneable {@@ -86,6 +94,7 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objec@Override public Object clone() {return new Segment((Chunk) this.chunk.clone(), this.start, this.length);}+ void force() { this.chunk.force(); }@Override public void close() throws IOException { this.chunk.close(); }}@@ -97,7 +106,45 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objecvoid set(long relPos, byte b);int length();Object clone();- @Override default void close() { /* no-op by default */ }+ default void force() { /* heap and read-only chunks need no synchronization */ }+ @Override default void close() throws IOException { /* no-op by default */ }+ }++ /**+ * One channel ownership shared by all segments of an appended file and by+ * read-only clones. A reference belongs to a ChunkedBytes instance, not to+ * every segment, so a multi-chunk file is closed exactly once per owner.+ */+ private static final class FileBacking implements Closeable {+ final FileChannel channel;+ private final AtomicInteger references;++ FileBacking(final FileChannel channel) {+ this.channel = channel;+ this.references = new AtomicInteger(1);+ }++ FileBacking retain() {+ int current;+ do {+ current = this.references.get();+ if (current <= 0) throw new IllegalStateException("File backing is closed");+ if (current == Integer.MAX_VALUE) {+ throw new IllegalStateException("Too many references to file backing");+ }+ } while (!this.references.compareAndSet(current, current + 1));+ return this;+ }++ @Override+ public void close() throws IOException {+ final int remaining = this.references.decrementAndGet();+ if (remaining == 0) {+ this.channel.close();+ } else if (remaining < 0) {+ throw new IOException("File backing closed more than once");+ }+ }}/** On-heap chunk. */@@ -131,14 +178,14 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objec/** File-backed chunk using MappedByteBuffer (lazy mapped). */private static final class FileChunk implements Chunk, Cloneable {- final FileChannel ch;+ final FileBacking backing;final long fileOffset; // offset in the file where this chunk startsfinal int len;final boolean writable;private volatile MappedByteBuffer mm; // lazily created- FileChunk(FileChannel ch, long fileOffset, int len, boolean writable) {- this.ch = ch; this.fileOffset = fileOffset; this.len = len; this.writable = writable;+ FileChunk(FileBacking backing, long fileOffset, int len, boolean writable) {+ this.backing = backing; this.fileOffset = fileOffset; this.len = len; this.writable = writable;}private MappedByteBuffer map() {MappedByteBuffer local = this.mm;@@ -148,9 +195,9 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objecif (local == null) {final FileChannel.MapMode mapMode = this.writable ? FileChannel.MapMode.READ_WRITE : FileChannel.MapMode.READ_ONLY;try {- this.mm = local = this.ch.map(mapMode, this.fileOffset, this.len);+ this.mm = local = this.backing.channel.map(mapMode, this.fileOffset, this.len);} catch (final IOException e) {- throw new RuntimeException(e);+ throw new UncheckedIOException(e);}}}@@ -166,7 +213,7 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objecreturn take;}@Override public int write(long p, byte[] src, int off, int len) {- if (!this.writable) throw new RuntimeException("FileChunk is read-only");+ if (!this.writable) throw new ReadOnlyBufferException();if (p >= this.len) return -1;final int take = Math.min(len, this.len - (int)p);final ByteBuffer dup = this.map().duplicate();@@ -178,18 +225,30 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objecreturn this.map().get((int)p);}@Override public void set(long p, byte b) {- if (!this.writable) throw new RuntimeException("FileChunk is read-only");+ if (!this.writable) throw new ReadOnlyBufferException();this.map().put((int)p, b);}@Override public int length() { return this.len; }- @Override public Object clone() { return new FileChunk(this.ch, this.fileOffset, this.len, this.writable); }- @Override public void close() {- // Best-effort explicit unmap to release file handles promptly.+ @Override public Object clone() { return new FileChunk(this.backing, this.fileOffset, this.len, this.writable); }+ @Override public void force() {final MappedByteBuffer local = this.mm;+ if (this.writable && local != null) local.force();+ }+ @Override public void close() throws IOException {+ /* Clear the reference before cleaning so close cannot clean it twice. */+ final MappedByteBuffer local;+ synchronized (this) {+ local = this.mm;+ this.mm = null;+ }if (local != null) {- try { Unmapper.unmap(local); } catch (final Throwable ignored) {}+ try {+ Unmapper.unmap(local);+ } catch (final Exception e) {+ throw new IOException("Cannot unmap file chunk at " + this.fileOffset, e);+ }}- // Channel is closed by owner (we don’t own it here).+ // The shared FileBacking is released by the owning ChunkedBytes.}}@@ -203,7 +262,7 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objecfinal var theUnsafe = unsafeClass.getDeclaredField("theUnsafe");theUnsafe.setAccessible(true);final Object unsafe = theUnsafe.get(null);- unsafeClass.getMethod("invokeCleaner", MappedByteBuffer.class).invoke(unsafe, bb);+ unsafeClass.getMethod("invokeCleaner", ByteBuffer.class).invoke(unsafe, bb);return;} catch (final Throwable ignore) {}// Java 8: DirectBuffer.cleaner().clean()@@ -218,22 +277,38 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objec// ------------------- public API -------------------/** Total logical size in bytes. */- public long size() { return this.size; }+ public long size() {+ requireOpen();+ return this.size;+ }/** Append bytes from an InputStream into on-heap chunks. */- public void writeFrom(InputStream in) throws IOException {+ public void writeFrom(final InputStream in) throws IOException {+ requireOpen();+ if (in == null) throw new NullPointerException("in");final byte[] tmp = new byte[64 * 1024];int r;while ((r = in.read(tmp)) != -1) {// InputStream.read(byte[]) may legally return 0if (r == 0) continue;+ /*+ * InputStream.read() is foreign code and must run without holding the+ * ChunkedBytes monitor. Otherwise an input implementation which owns+ * another lock can invert that lock order when its owner concurrently+ * calls append() or close(). append() provides the required, narrowly+ * scoped synchronization for the actual state change.+ */this.append(tmp, 0, r);}}/** Append a byte array (copied into on-heap chunks). */- public void append(byte[] src, int off, int len) {+ public synchronized void append(final byte[] src, int off, int len) {+ requireOpen();+ checkBounds(src, off, len, "src");+ if (len == 0) return;+ Math.addExact(this.size, len);while (len > 0) {int space = this.spaceInTailHeapChunk();if (space == 0) {@@ -256,51 +331,65 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objec}}- public void append(byte[] src) {+ public void append(final byte[] src) {+ if (src == null) throw new NullPointerException("src");this.append(src, 0, src.length);}- public void append(String src, int off, int len) {+ public void append(final String src, final int off, final int len) {+ if (src == null) throw new NullPointerException("src");this.append(src.substring(off, off + len));}- public void append(String src) {+ public void append(final String src) {+ if (src == null) throw new NullPointerException("src");this.append(UTF8.getBytes(src));}/** Adopt an entire file as zero-copy file-backed segments (read-only). */- public void appendFile(Path path) {+ public void appendFile(final Path path) {this.appendFile(path, false);}/** Adopt a file; set writable=true to allow modifications to the mapped bytes. */- public void appendFile(Path path, boolean writable) {+ public synchronized void appendFile(final Path path, final boolean writable) {+ requireOpen();+ if (path == null) throw new NullPointerException("path");FileChannel ch = null;try {ch = FileChannel.open(path, writable? new OpenOption[]{StandardOpenOption.READ, StandardOpenOption.WRITE}: new OpenOption[]{StandardOpenOption.READ});final long fileSize = ch.size();+ if (fileSize == 0) return;+ final long newSize = Math.addExact(this.size, fileSize);+ final FileBacking backing = new FileBacking(ch);+ final List<Segment> fileSegments = new ArrayList<>();long pos = 0;while (pos < fileSize) {final int len = (int)Math.min(CHUNK_SIZE, fileSize - pos);- this.segments.add(new Segment(new FileChunk(ch, pos, len, writable), this.size, len));- this.size += len;+ fileSegments.add(new Segment(+ new FileChunk(backing, pos, len, writable), this.size + pos, len));pos += len;}- // We keep the FileChannel open until this ChunkedBytes is closed.- this.fileChannels.add(ch);+ this.segments.addAll(fileSegments);+ this.size = newSize;+ // The shared backing owns the channel until this instance and all clones close.+ this.fileBackings.add(backing);ch = null; // prevent closing in finally} catch (final IOException e) {- throw new RuntimeException(e);+ throw new UncheckedIOException(e);} finally {if (ch != null) try { ch.close(); } catch (final IOException ignore) {}}}/** Read into dst starting at logical position pos. Returns bytes read or -1 at EOF. */- public int read(long pos, byte[] dst, int off, int len) {+ public int read(final long pos, final byte[] dst, final int off, final int len) {+ requireOpen();if (pos < 0) throw new IllegalArgumentException("pos < 0");+ checkBounds(dst, off, len, "dst");+ if (len == 0) return 0;if (pos >= this.size) return -1;long remaining = Math.min(len, this.size - pos);int done = 0;@@ -319,8 +408,11 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objec}/** Write bytes at logical position pos (requires writable backing for those ranges). */- public int write(long pos, byte[] src, int off, int len) {+ public int write(final long pos, final byte[] src, final int off, final int len) {+ requireOpen();if (pos < 0) throw new IllegalArgumentException("pos < 0");+ checkBounds(src, off, len, "src");+ if (len == 0) return 0;if (pos >= this.size) return -1;long remaining = Math.min(len, this.size - pos);int done = 0; int idx = this.findSegment(pos); long p = pos;@@ -338,31 +430,48 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objec/** InputStream view (no copying), supports >2 GB. */public InputStream openStream() {+ requireOpen();return new InputStream() {long pos = 0;+ boolean streamClosed = false;+ private void requireStreamOpen() throws IOException {+ if (this.streamClosed) throw new IOException("Stream is closed");+ }@Override public int read() throws IOException {+ requireStreamOpen();final byte[] one = new byte[1];final int n = this.read(one, 0, 1);return n < 0 ? -1 : (one[0] & 0xFF);}@Override public int read(byte[] b, int off, int len) throws IOException {+ requireStreamOpen();final int n = ChunkedBytes.this.read(this.pos, b, off, len);if (n > 0) this.pos += n;return n;}- @Override public long skip(long n) {+ @Override public long skip(long n) throws IOException {+ requireStreamOpen();+ ChunkedBytes.this.requireOpen();+ if (n <= 0) return 0;final long k = Math.min(n, ChunkedBytes.this.size - this.pos);this.pos += k; return k;}- @Override public int available() {+ @Override public int available() throws IOException {+ requireStreamOpen();+ ChunkedBytes.this.requireOpen();final long rem = ChunkedBytes.this.size - this.pos;return rem > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) rem;}+ @Override public void close() {+ this.streamClosed = true;+ }};}/** Write the whole content to an OutputStream. */- public void writeTo(OutputStream out) {+ public void writeTo(final OutputStream out) {+ requireOpen();+ if (out == null) throw new NullPointerException("out");final byte[] tmp = new byte[256 * 1024];long p = 0;try {@@ -373,13 +482,14 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objecp += n;}} catch (final IOException e) {- throw new RuntimeException(e);+ throw new UncheckedIOException(e);}}/** Materialize as a single byte[] (only if total size fits in an int). */public byte[] toByteArray() {- if (this.size > Integer.MAX_VALUE) throw new RuntimeException("Size > Integer.MAX_VALUE");+ requireOpen();+ if (this.size > Integer.MAX_VALUE) throw new IllegalStateException("Size > Integer.MAX_VALUE");final byte[] all = new byte[(int) this.size];this.writeTo(new ByteArrayOutputStream() {int offset = 0;@@ -393,6 +503,7 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objec@Overridepublic synchronized void write(int b) {+ requireOpen();// Append single byte at end (OutputStream semantics)final byte[] one = new byte[] { (byte) b };this.append(one, 0, 1);@@ -400,33 +511,63 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objec@Overridepublic synchronized void write(byte[] b, int off, int len) {- if (b == null) throw new NullPointerException("b");- if (off < 0 || len < 0 || off + len > b.length) throw new IndexOutOfBoundsException();+ requireOpen();+ checkBounds(b, off, len, "b");// Append to end; grows with heap chunks as neededthis.append(b, off, len);}public void writeBytes(byte[] b) {+ if (b == null) throw new NullPointerException("b");this.write(b, 0, b.length);}@Overridepublic void flush() {- // No-op for heap chunks; file-backed segments via MappedByteBuffer are- // written eagerly. If you need a hard sync to disk, expose a separate- // method to force() mapped buffers.+ force();+ }++ /** Force changes in every writable mapped chunk to its storage device. */+ public synchronized void force() {+ requireOpen();+ for (final Segment s : this.segments) {+ s.force();+ }}- @Override public void close() {- // Close/unmap all segments and channels.- Exception first = null;+ @Override+ public synchronized void close() {+ if (this.closed) return;++ Exception failure = null;+ try {+ /* Mapped writes must be forced before the buffers are unmapped. */+ force();+ } catch (final RuntimeException e) {+ failure = e;+ }+ this.closed = true;+for (final Segment s : this.segments) {- try { s.close(); } catch (final Exception e) { if (first == null) first = e; }+ try {+ s.close();+ } catch (final Exception e) {+ failure = addFailure(failure, e);+ }}- for (final FileChannel ch : this.fileChannels) {- try { ch.close(); } catch (final Exception e) { if (first == null) first = e; }+ for (final FileBacking backing : this.fileBackings) {+ try {+ backing.close();+ } catch (final Exception e) {+ failure = addFailure(failure, e);+ }+ }+ if (failure instanceof UncheckedIOException) throw (UncheckedIOException) failure;+ if (failure instanceof IOException) {+ throw new UncheckedIOException("Cannot close ChunkedBytes", (IOException) failure);}- if (first != null) throw new RuntimeException(first.getMessage());+ if (failure instanceof RuntimeException) throw (RuntimeException) failure;+ if (failure != null) throw new RuntimeException("Cannot close ChunkedBytes", failure);}@Override@@ -435,8 +576,9 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objec}public byte get(long pos) {+ requireOpen();if (pos < 0) throw new IllegalArgumentException("pos < 0");- if (pos >= this.size) throw new RuntimeException("pos >= size");+ if (pos >= this.size) throw new IndexOutOfBoundsException("pos >= size");final int idx = this.findSegment(pos);final Segment s = this.segments.get(idx);final long rel = pos - s.start;@@ -444,11 +586,22 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objec}@Override- public Object clone() {- List<Segment> list = new ArrayList<>(this.segments.size());- for (Segment s: this.segments) list.add((Segment) s.clone());- final ChunkedBytes cb = new ChunkedBytes(list, this.size);- return cb;+ public synchronized Object clone() {+ requireOpen();+ final List<FileBacking> retainedBackings = new ArrayList<>(this.fileBackings.size());+ try {+ for (final FileBacking backing : this.fileBackings) {+ retainedBackings.add(backing.retain());+ }+ final List<Segment> clonedSegments = new ArrayList<>(this.segments.size());+ for (final Segment segment : this.segments) {+ clonedSegments.add((Segment) segment.clone());+ }+ return new ChunkedBytes(clonedSegments, this.size, retainedBackings);+ } catch (final RuntimeException | Error e) {+ releaseRetainedBackings(retainedBackings, e);+ throw e;+ }}@Override@@ -457,44 +610,47 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objecif (o instanceof ChunkedBytes) {final ChunkedBytes cb = (ChunkedBytes)o;if (this.size != cb.size) return false;- for (long i = 0; i < this.size; i++) {- if (this.get(i) != cb.get(i)) return false;- }- return true;- }- if (o instanceof byte[]) {- final byte[] b = (byte[])o;- if (this.size != b.length) return false;- for (int i = 0; i < b.length; i++) {- if (this.get(i) != b[i]) return false;- }- return true;- }- if (o instanceof String) {- return this.equals(UTF8.getBytes((String) o));+ return compareChunked(cb) == 0;}return false;}+ @Override+ public int hashCode() {+ requireOpen();+ int result = 1;+ final byte[] buffer = new byte[64 * 1024];+ long position = 0;+ while (position < this.size) {+ final int length = (int) Math.min(buffer.length, this.size - position);+ final int read = this.read(position, buffer, 0, length);+ if (read != length) throw new IllegalStateException("Short read while hashing");+ for (int i = 0; i < read; i++) result = 31 * result + buffer[i];+ position += read;+ }+ return result;+ }++ /** Content comparison retained for callers which previously used equals(byte[]). */+ public boolean contentEquals(final byte[] bytes) {+ requireOpen();+ if (bytes == null || this.size != bytes.length) return false;+ return compareBytes(bytes) == 0;+ }++ /** Content comparison retained for callers which previously used equals(String). */+ public boolean contentEquals(final String text) {+ requireOpen();+ return text != null && contentEquals(UTF8.getBytes(text));+ }+@Overridepublic int compareTo(Object o) {if (o instanceof ChunkedBytes) {- final ChunkedBytes cb = (ChunkedBytes)o;- final int minLen = (int)Math.min(this.size, cb.size);- for (int i = 0; i < minLen; i++) {- final int diff = (this.get(i) & 0xFF) - (cb.get(i) & 0xFF);- if (diff != 0) return diff;- }- return Long.compare(this.size, cb.size);+ return compareChunked((ChunkedBytes) o);}if (o instanceof byte[]) {- final byte[] b = (byte[])o;- final int minLen = (int)Math.min(this.size, b.length);- for (int i = 0; i < minLen; i++) {- final int diff = (this.get(i) & 0xFF) - (b[i] & 0xFF);- if (diff != 0) return diff;- }- return Long.compare(this.size, b.length);+ return compareBytes((byte[]) o);}if (o instanceof String) {return this.compareTo(UTF8.getBytes((String) o));@@ -504,7 +660,81 @@ public final class ChunkedBytes extends OutputStream implements Comparable<Objec// ------------------- internals -------------------- private final List<FileChannel> fileChannels = new ArrayList<>();+ private final List<FileBacking> fileBackings = new ArrayList<>();++ private void requireOpen() {+ if (this.closed) throw new IllegalStateException("ChunkedBytes is closed");+ }++ private static void checkBounds(final byte[] bytes, final int offset,+ final int length, final String name) {+ if (bytes == null) throw new NullPointerException(name);+ if (offset < 0 || length < 0 || offset > bytes.length - length) {+ throw new IndexOutOfBoundsException(+ "offset=" + offset + ", length=" + length+ + ", array length=" + bytes.length);+ }+ }++ private static Exception addFailure(final Exception current, final Exception added) {+ if (current == null) return added;+ current.addSuppressed(added);+ return current;+ }++ private static void releaseRetainedBackings(+ final List<FileBacking> retainedBackings,+ final Throwable failure) {+ for (final FileBacking backing : retainedBackings) {+ try {+ backing.close();+ } catch (final IOException closeFailure) {+ failure.addSuppressed(closeFailure);+ }+ }+ }++ private int compareChunked(final ChunkedBytes other) {+ requireOpen();+ other.requireOpen();+ final long minimumLength = Math.min(this.size, other.size);+ final byte[] left = new byte[64 * 1024];+ final byte[] right = new byte[left.length];+ long position = 0;+ while (position < minimumLength) {+ final int length = (int) Math.min(left.length, minimumLength - position);+ if (this.read(position, left, 0, length) != length+ || other.read(position, right, 0, length) != length) {+ throw new IllegalStateException("Short read while comparing ChunkedBytes");+ }+ for (int i = 0; i < length; i++) {+ final int difference = (left[i] & 0xff) - (right[i] & 0xff);+ if (difference != 0) return difference;+ }+ position += length;+ }+ return Long.compare(this.size, other.size);+ }++ private int compareBytes(final byte[] bytes) {+ requireOpen();+ final long minimumLength = Math.min(this.size, bytes.length);+ final byte[] left = new byte[64 * 1024];+ long position = 0;+ while (position < minimumLength) {+ final int length = (int) Math.min(left.length, minimumLength - position);+ if (this.read(position, left, 0, length) != length) {+ throw new IllegalStateException("Short read while comparing bytes");+ }+ final int byteOffset = (int) position;+ for (int i = 0; i < length; i++) {+ final int difference = (left[i] & 0xff) - (bytes[byteOffset + i] & 0xff);+ if (difference != 0) return difference;+ }+ position += length;+ }+ return Long.compare(this.size, bytes.length);+ }private int findSegment(long pos) {int lo = 0, hi = this.segments.size() - 1;diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.javaindex 3e0194b06..8543d4ec5 100644--- a/source/net/yacy/kelondro/blob/ArrayStack.java+++ b/source/net/yacy/kelondro/blob/ArrayStack.java@@ -95,6 +95,7 @@ public class ArrayStack implements BLOB {private long repositoryAgeMax;private long repositorySizeMax;private List<blobItem> blobs;+ private blobItem writableBlob;private final String prefix;private final int buffersize;private final boolean trimall;@@ -123,6 +124,7 @@ public class ArrayStack implements BLOB {this.fileSizeLimit = maxFileSize;this.repositoryAgeMax = Long.MAX_VALUE;this.repositorySizeMax = Long.MAX_VALUE;+ this.writableBlob = null;this.trimall = trimall;// init the thread pool for the keeperOf executor service@@ -182,7 +184,6 @@ public class ArrayStack implements BLOB {// find maximum time: the file with this time will be given a write bufferfinal TreeMap<Long, blobItem> sortedItems = new TreeMap<Long, blobItem>();- BLOB oneBlob;File f;long maxtime = 0;for (final String file : files) {@@ -203,13 +204,17 @@ public class ArrayStack implements BLOB {f = new File(heapLocation, file);time = d.getTime();try {+ final blobItem item;if (time == maxtime && !trimall) {- oneBlob = new Heap(f, keylength, ordering, buffersize);+ final Heap heap = new Heap(f, keylength, ordering, buffersize);+ item = new blobItem(d, f, heap);+ this.writableBlob = item;} else {- oneBlob = new HeapModifier(f, keylength, ordering);- oneBlob.optimize(); // no writings here, can be used with minimum memory+ final ImmutableBLOB blob = new HeapModifier(f, keylength, ordering);+ blob.optimize(); // no writings here, can be used with minimum memory+ item = new blobItem(d, f, blob);}- sortedItems.put(Long.valueOf(time), new blobItem(d, f, oneBlob));+ sortedItems.put(Long.valueOf(time), item);} catch (final IOException e) {if (deleteonfail) {ConcurrentLog.warn("KELONDRO", "ArrayStack: cannot read file " + f.getName() + ", deleting it (smart fail; alternative would be: crash; required user action would be same as deletion)");@@ -257,14 +262,18 @@ public class ArrayStack implements BLOB {} catch (final ParseException e) {throw new IOException("date parse problem with file " + location.toString() + ": " + e.getMessage());}- BLOB oneBlob;- if (full && this.buffersize > 0 && !this.trimall) {- oneBlob = new Heap(location, this.keylength, this.ordering, this.buffersize);+ final blobItem item;+ if (full && !this.trimall) {+ sealWriter();+ final Heap heap = new Heap(location, this.keylength, this.ordering, this.buffersize);+ item = new blobItem(d, location, heap);+ this.writableBlob = item;} else {- oneBlob = new HeapModifier(location, this.keylength, this.ordering);- oneBlob.optimize();+ final ImmutableBLOB blob = new HeapModifier(location, this.keylength, this.ordering);+ blob.optimize();+ item = new blobItem(d, location, blob);}- this.blobs.add(new blobItem(d, location, oneBlob));+ this.blobs.add(item);}private synchronized void unmountBLOB(final File location, final boolean writeIDX) {@@ -273,6 +282,7 @@ public class ArrayStack implements BLOB {b = this.blobs.get(i);if (b.location.getAbsolutePath().equals(location.getAbsolutePath())) {this.blobs.remove(i);+ clearWritableReference(b);b.blob.close(writeIDX);b.blob = null;b.location = null;@@ -284,6 +294,7 @@ public class ArrayStack implements BLOB {private File unmount(final int idx) {final blobItem b = this.blobs.remove(idx);+ clearWritableReference(b);b.blob.close(false);b.blob = null;final File f = b.location;@@ -413,6 +424,7 @@ public class ArrayStack implements BLOB {while (!this.blobs.isEmpty() && System.currentTimeMillis() - this.blobs.get(0).creation.getTime() - this.fileAgeLimit > this.repositoryAgeMax) {// too oldfinal blobItem oldestBLOB = this.blobs.remove(0);+ clearWritableReference(oldestBLOB);oldestBLOB.blob.close(false);oldestBLOB.blob = null;FileUtils.deletedelete(oldestBLOB.location);@@ -422,6 +434,7 @@ public class ArrayStack implements BLOB {while (!this.blobs.isEmpty() && length() > this.repositorySizeMax) {// too largefinal blobItem oldestBLOB = this.blobs.remove(0);+ clearWritableReference(oldestBLOB);oldestBLOB.blob.close(false);FileUtils.deletedelete(oldestBLOB.location);}@@ -445,18 +458,89 @@ public class ArrayStack implements BLOB {private class blobItem {Date creation;File location;- BLOB blob;+ ImmutableBLOB blob;+ BLOB writer;++ public blobItem(final Date creation, final File location, final ImmutableBLOB blob) {+ assert blob != null;+ this.creation = creation;+ this.location = location;+ this.blob = blob;+ this.writer = null;+ }+public blobItem(final Date creation, final File location, final BLOB blob) {assert blob != null;this.creation = creation;this.location = location;this.blob = blob;+ this.writer = blob;}+public blobItem(final int buffer) throws IOException {// make a new blob file and assign it in this itemthis.creation = new Date();this.location = newBLOB(this.creation);- this.blob = (buffer == 0) ? new HeapModifier(this.location, ArrayStack.this.keylength, ArrayStack.this.ordering) : new Heap(this.location, ArrayStack.this.keylength, ArrayStack.this.ordering, buffer);+ final Heap heap = new Heap(this.location, ArrayStack.this.keylength, ArrayStack.this.ordering, buffer);+ this.blob = heap;+ this.writer = heap;+ }+ }++ private void clearWritableReference(final blobItem item) {+ if (item == this.writableBlob) this.writableBlob = null;+ item.writer = null;+ }++ /**+ * Close the current writable heap and reopen it with only the capabilities+ * required by a sealed segment.+ */+ private void sealWriter() throws IOException {+ final blobItem item = this.writableBlob;+ if (item == null) return;+ if (item.writer == null) {+ this.writableBlob = null;+ return;+ }++ try {+ item.writer.close(true);+ } catch (final RuntimeException failure) {+ /*+ * HeapReader.close() releases the file, index and gap structures even+ * when publishing their fingerprint pair fails. Keeping that Heap as+ * writer would therefore leave ArrayStack pointing at a closed object.+ * Reopen the authoritative blob before reporting the publish failure,+ * so the caller may retry or continue writing to this ArrayStack.+ */+ restoreWriterAfterFailedSeal(item, failure);+ throw failure;+ }+ item.writer = null;+ this.writableBlob = null;+ try {+ final ImmutableBLOB sealed = new HeapModifier(item.location, this.keylength, this.ordering);+ sealed.optimize();+ item.blob = sealed;+ } catch (final IOException | RuntimeException failure) {+ restoreWriterAfterFailedSeal(item, failure);+ throw failure;+ }+ }++ /** Restore the mutable state after either phase of sealing has failed. */+ private void restoreWriterAfterFailedSeal(+ final blobItem item, final Throwable failure) {+ clearWritableReference(item);+ try {+ final Heap restored = new Heap(+ item.location, this.keylength, this.ordering, this.buffersize);+ item.blob = restored;+ item.writer = restored;+ this.writableBlob = item;+ } catch (final IOException | RuntimeException restoreFailure) {+ failure.addSuppressed(restoreFailure);}}@@ -478,6 +562,7 @@ public class ArrayStack implements BLOB {for (final blobItem bi: this.blobs) {bi.blob.clear();bi.blob.close(false);+ clearWritableReference(bi);HeapWriter.delete(bi.location);}this.blobs.clear();@@ -695,7 +780,7 @@ public class ArrayStack implements BLOB {@Overrideprotected byte[] next0() {while (this.bii.hasNext()) {- final BLOB b = this.bii.next().blob;+ final ImmutableBLOB b = this.bii.next().blob;if (b == null) continue;try {final byte[] n = b.get(this.key);@@ -752,7 +837,7 @@ public class ArrayStack implements BLOB {@Overrideprotected Long next0() {while (this.bii.hasNext()) {- final BLOB b = this.bii.next().blob;+ final ImmutableBLOB b = this.bii.next().blob;if (b == null) continue;try {final long l = b.length(this.key);@@ -789,22 +874,18 @@ public class ArrayStack implements BLOB {*/@Overridepublic synchronized void insert(final byte[] key, final byte[] b) throws IOException {- blobItem bi = (this.blobs.isEmpty()) ? null : this.blobs.get(this.blobs.size() - 1);- /*- if (bi == null)- System.out.println("bi == null");- else if (System.currentTimeMillis() - bi.creation.getTime() > this.fileAgeLimit)- System.out.println("System.currentTimeMillis() - bi.creation.getTime() > this.maxage");- else if (bi.location.length() > this.fileSizeLimit)- System.out.println("bi.location.length() > this.maxsize");- */- if ((bi == null) || (System.currentTimeMillis() - bi.creation.getTime() > this.fileAgeLimit) || (bi.location.length() > this.fileSizeLimit && this.fileSizeLimit >= 0)) {+ blobItem bi = this.writableBlob;+ if (bi == null+ || System.currentTimeMillis() - bi.creation.getTime() > this.fileAgeLimit+ || bi.location.length() > this.fileSizeLimit && this.fileSizeLimit >= 0) {+ sealWriter();// add a new blob to the arraybi = new blobItem(this.buffersize);this.blobs.add(bi);+ this.writableBlob = bi;}- assert bi.blob instanceof Heap;- bi.blob.insert(key, b);+ assert bi.writer != null;+ bi.writer.insert(key, b);executeLimits();}@@ -816,9 +897,12 @@ public class ArrayStack implements BLOB {*/@Overridepublic synchronized int replace(final byte[] key, final Rewriter rewriter) throws IOException, SpaceExceededException {+ final Reducer reducer = rewriter::rewrite;int d = 0;for (final blobItem bi: this.blobs) {- d += bi.blob.replace(key, rewriter);+ d += bi.writer == null+ ? bi.blob.reduce(key, reducer)+ : bi.writer.replace(key, rewriter);}return d;}@@ -833,7 +917,9 @@ public class ArrayStack implements BLOB {public synchronized int reduce(final byte[] key, final Reducer reduce) throws IOException, SpaceExceededException {int d = 0;for (final blobItem bi: this.blobs) {- d += bi.blob.reduce(key, reduce);+ d += bi.writer == null+ ? bi.blob.reduce(key, reduce)+ : bi.writer.replace(key, reduce);}return d;}@@ -903,10 +989,20 @@ public class ArrayStack implements BLOB {*/@Overridepublic synchronized void close(final boolean writeIDX) {- for (final blobItem bi: this.blobs) bi.blob.close(writeIDX);+ RuntimeException failure = null;+ for (final blobItem bi: this.blobs) {+ clearWritableReference(bi);+ try {+ bi.blob.close(writeIDX);+ } catch (final RuntimeException closeFailure) {+ if (failure == null) failure = closeFailure;+ else failure.addSuppressed(closeFailure);+ }+ }this.blobs.clear();this.blobs = null;this.executor.shutdown();+ if (failure != null) throw failure;}/**diff --git a/source/net/yacy/kelondro/blob/BLOB.java b/source/net/yacy/kelondro/blob/BLOB.javaindex aee7b19a5..1dff4b2a8 100644--- a/source/net/yacy/kelondro/blob/BLOB.java+++ b/source/net/yacy/kelondro/blob/BLOB.java@@ -4,10 +4,6 @@//// This is a part of YaCy, a peer-to-peer based web search engine//-// $LastChangedDate$-// $LastChangedRevision$-// $LastChangedBy$-//// LICENSE//// This program is free software; you can redistribute it and/or modify@@ -28,120 +24,10 @@ package net.yacy.kelondro.blob;import java.io.IOException;-import net.yacy.cora.order.ByteOrder;-import net.yacy.cora.order.CloneableIterator;import net.yacy.cora.util.SpaceExceededException;-public interface BLOB {-- /**- * return a name of the BLOB; can be the file name- * @return- */- public String name();-- /**- * ask for the length of the primary key- * @return the length of the key- */- public int keylength();-- /**- * return the underlying odering- * @return- */- public ByteOrder ordering();-- /**- * clears the content of the database- * @throws IOException- */- public void clear() throws IOException;-- /**- * trim the index of the database: this releases memory not currently used- * @throws IOException- */- public void optimize();-- /**- * calculate the memory in RAM that the BLOB occupies- * @return number of bytes that is used- */- public long mem();-- /**- * ask for the number of entries- * @return the number of entries in the table- */- public int size();-- /**- * ask if the BLOB is empty- * @return true iff size() == 0- */- public boolean isEmpty();-- /**- * iterator over all keys- * @param up- * @param rotating- * @return- * @throws IOException- */- public CloneableIterator<byte[]> keys(boolean up, boolean rotating) throws IOException;-- /**- * iterate over all keys- * @param up- * @param firstKey- * @return- * @throws IOException- */- public CloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException;-- /**- * check if a specific key is in the database- * @param key the primary key- * @return- * @throws IOException- */- public boolean containsKey(byte[] key);-- /**- * retrieve the whole BLOB from the table- * @param key the primary key- * @return- * @throws IOException- */- public byte[] get(byte[] key) throws IOException, SpaceExceededException;- public byte[] get(Object key);-- /**- * retrieve the size of the database- * @param- * @return the size of the database in bytes- * @throws IOException- */- public long length() throws IOException;-- /**- * retrieve the size of the BLOB- * @param key- * @return the size of the BLOB or -1 if the BLOB does not exist- * @throws IOException- */- public long length(byte[] key) throws IOException;-- /**- * write a whole byte array as BLOB to the table- * @param key the primary key- * @param b- * @throws IOException- * @throws SpaceExceededException- */- public void insert(byte[] key, byte[] b) throws IOException;+public interface BLOB extends ImmutableBLOB {/*** replace an existing entry in the BLOB with a new entry@@ -160,49 +46,12 @@ public interface BLOB {public int replace(byte[] key, Rewriter rewriter) throws IOException, SpaceExceededException;/**- * a reduce method is the same as the replace. A replace subsumes a reduce method. A reduce method may be more simple.+ * write a whole byte array as BLOB to the table* @param key the primary key- * @param reducer- * @return the number of bytes that the rewriter reduced the BLOB+ * @param b* @throws IOException* @throws SpaceExceededException*/- public int reduce(byte[] key, Reducer reducer) throws IOException, SpaceExceededException;-- /**- * remove a BLOB- * @param key the primary key- * @throws IOException- */- public void delete(byte[] key) throws IOException;-- /**- * close the BLOB table- */- public void close(boolean writeIDX);-- public interface Rewriter {-- /**- * a rewrite method that is used in the replace functionality of a BLOB- * the result of such a rewrite must be always smaller or equal in size- * of the input- * @param b- * @return an array that is equal or smaller in size than b- * @throws SpaceExceededException- */- public byte[] rewrite(byte[] b) throws SpaceExceededException;-- }-- public interface Reducer extends Rewriter {-- /**- * A Reducer is a rewriter that reduced the content. There are no additional methods in this interface.- * The interface shall be used in replacement of a Rewriter to simply state the fact that the rewritement- * also reduces the content of a BLOB entry or may also keep the size the same;- */-- }+ public void insert(byte[] key, byte[] b) throws IOException;}diff --git a/source/net/yacy/kelondro/blob/Gap.java b/source/net/yacy/kelondro/blob/Gap.javaindex 2d5cc08f0..c6e65d24e 100644--- a/source/net/yacy/kelondro/blob/Gap.java+++ b/source/net/yacy/kelondro/blob/Gap.java@@ -34,6 +34,9 @@ import java.io.File;import java.io.FileInputStream;import java.io.FileOutputStream;import java.io.IOException;+import java.nio.file.AtomicMoveNotSupportedException;+import java.nio.file.Files;+import java.nio.file.StandardCopyOption;import java.util.Iterator;import java.util.Map;import java.util.TreeMap;@@ -101,36 +104,47 @@ public class Gap extends TreeMap<Long, Integer> {* @return* @throws IOException*/- public int dump(File file) throws IOException {- File tmp = new File(file.getParentFile(), file.getName() + ".prt");- Iterator<Map.Entry<Long, Integer>> i = this.entrySet().iterator();+ public int dump(final File file) throws IOException {+ final File tmp = new File(file.getParentFile(), file.getName() + ".prt");+ final Iterator<Map.Entry<Long, Integer>> i = this.entrySet().iterator();DataOutputStream os;int c = 0;- try (/* Resource automatically closed by this try-with-resources statement */- final FileOutputStream fileStream = new FileOutputStream(tmp);- ) {- try {- os = new DataOutputStream(new BufferedOutputStream(fileStream, (Integer.SIZE + Long.SIZE) * 1024)); // = 16*1024*recordsize- } catch (final OutOfMemoryError e) {- os = new DataOutputStream(fileStream);- }- try {- Map.Entry<Long, Integer> e;- while (i.hasNext()) {- e = i.next();- os.writeLong(e.getKey().longValue());- os.writeInt(e.getValue().intValue());- c++;- }- os.flush();- } finally {- os.close();- }+ try {+ try (final FileOutputStream fileStream = new FileOutputStream(tmp)) {+ try {+ os = new DataOutputStream(new BufferedOutputStream(+ fileStream, (Integer.SIZE + Long.SIZE) * 1024));+ } catch (final OutOfMemoryError e) {+ os = new DataOutputStream(fileStream);+ }+ try {+ while (i.hasNext()) {+ final Map.Entry<Long, Integer> entry = i.next();+ os.writeLong(entry.getKey().longValue());+ os.writeInt(entry.getValue().intValue());+ c++;+ }+ os.flush();+ } finally {+ os.close();+ }+ }+ try {+ Files.move(tmp.toPath(), file.toPath(),+ StandardCopyOption.ATOMIC_MOVE,+ StandardCopyOption.REPLACE_EXISTING);+ } catch (final AtomicMoveNotSupportedException e) {+ Files.move(tmp.toPath(), file.toPath(),+ StandardCopyOption.REPLACE_EXISTING);+ }+ } catch (final IOException e) {+ try {+ Files.deleteIfExists(tmp.toPath());+ } catch (final IOException cleanupFailure) {+ e.addSuppressed(cleanupFailure);+ }+ throw e;}- tmp.renameTo(file);- assert file.exists() : file.toString();- assert !tmp.exists() : tmp.toString();-return c;}diff --git a/source/net/yacy/kelondro/blob/Heap.java b/source/net/yacy/kelondro/blob/Heap.javaindex 203767e99..33dc82582 100644--- a/source/net/yacy/kelondro/blob/Heap.java+++ b/source/net/yacy/kelondro/blob/Heap.java@@ -28,6 +28,7 @@ package net.yacy.kelondro.blob;import java.io.File;import java.io.IOException;+import java.io.UncheckedIOException;import java.util.AbstractMap;import java.util.ArrayList;import java.util.HashMap;@@ -41,13 +42,14 @@ import net.yacy.cora.document.encoding.UTF8;import net.yacy.cora.order.ByteOrder;import net.yacy.cora.order.CloneableIterator;import net.yacy.cora.order.NaturalOrder;+import net.yacy.cora.storage.HandleMap;import net.yacy.cora.util.ConcurrentLog;import net.yacy.cora.util.SpaceExceededException;import net.yacy.kelondro.io.AbstractWriter;import net.yacy.kelondro.util.MemoryControl;-public final class Heap extends HeapModifier implements BLOB {+public final class Heap extends AbstractHeapModifier<HandleMap> implements BLOB {private SortedMap<byte[], byte[]> buffer; // a write buffer to limit IO to the fileprivate int buffersize; // bytes that are buffered in buffer@@ -88,7 +90,7 @@ public final class Heap extends HeapModifier implements BLOB {final int keylength,final ByteOrder ordering,int buffermax) throws IOException {- super(heapFile, keylength, ordering);+ super(heapFile, keylength, ordering, HeapReader.mutable());this.buffermax = buffermax;this.buffer = new TreeMap<byte[], byte[]>(ordering);this.buffersize = 0;@@ -344,20 +346,48 @@ public final class Heap extends HeapModifier implements BLOB {/*** close the BLOB table+ * @throws UncheckedIOException when buffered entries cannot be flushed*/@Overridepublic synchronized void close(final boolean writeIDX) {ConcurrentLog.info("KELONDRO", "Heap: closing heap " + this.name());+ IOException flushFailure = null;if (this.file != null && this.buffer != null) {try {flushBufferInternal();} catch (final IOException e) {- ConcurrentLog.logException(e);+ flushFailure = e;}}this.buffer = null;- super.close(writeIDX);++ Throwable closeFailure = null;+ try {+ /*+ * A failed flush may have changed the in-memory index without writing+ * every corresponding record. Publishing that index would turn a+ * recoverable I/O failure into a persistent, invalid fingerprint.+ * Closing without a dump makes the next opener rebuild from the blob.+ */+ super.close(writeIDX && flushFailure == null);+ } catch (final Throwable failure) {+ closeFailure = failure;+ }assert this.file == null;++ if (flushFailure != null) {+ if (closeFailure != null) flushFailure.addSuppressed(closeFailure);+ throw new UncheckedIOException(+ "Cannot flush heap buffer before closing " + this.name(),+ flushFailure);+ }+ if (closeFailure instanceof RuntimeException) {+ throw (RuntimeException) closeFailure;+ }+ if (closeFailure instanceof Error) throw (Error) closeFailure;+ if (closeFailure != null) {+ throw new IllegalStateException("Cannot close heap " + this.name(), closeFailure);+ }}@Override@@ -369,6 +399,13 @@ public final class Heap extends HeapModifier implements BLOB {return this.buffermax;}+ @Override+ public synchronized int replace(final byte[] key, final Rewriter rewriter) throws IOException, SpaceExceededException {+ /* The shared in-place implementation operates on persisted records. */+ flushBufferInternal();+ return rewrite(key, rewriter);+ }+/*** write a whole byte array as BLOB to the table* @param key the primary keydiff --git a/source/net/yacy/kelondro/blob/HeapModifier.java b/source/net/yacy/kelondro/blob/HeapModifier.javaindex ea22d47da..592b1e6dd 100644--- a/source/net/yacy/kelondro/blob/HeapModifier.java+++ b/source/net/yacy/kelondro/blob/HeapModifier.java@@ -30,14 +30,24 @@ import java.util.SortedMap;import net.yacy.cora.document.encoding.UTF8;import net.yacy.cora.order.ByteOrder;+import net.yacy.cora.storage.ImmutableHandleMap;import net.yacy.cora.util.ConcurrentLog;import net.yacy.cora.util.SpaceExceededException;+import net.yacy.kelondro.index.SSTableHandleMap;import net.yacy.kelondro.io.CachedFileWriter;import net.yacy.kelondro.util.FileUtils;import net.yacy.kelondro.util.MemoryControl;-public class HeapModifier extends HeapReader implements BLOB {+public class HeapModifier extends AbstractHeapModifier<SSTableHandleMap> {++ public HeapModifier(final File heapFile, final int keylength, final ByteOrder ordering) throws IOException {+ super(heapFile, keylength, ordering, HeapReader.immutable());+ }++}++abstract class AbstractHeapModifier<I extends ImmutableHandleMap> extends HeapReader<I> implements ImmutableBLOB {/** This class adds a remove operation to a BLOBHeapReader. That means that a BLOBModifier can@@ -54,8 +64,12 @@ public class HeapModifier extends HeapReader implements BLOB {* @param ordering* @throws IOException*/- public HeapModifier(final File heapFile, final int keylength, final ByteOrder ordering) throws IOException {- super(heapFile, keylength, ordering);+ protected AbstractHeapModifier(+ final File heapFile,+ final int keylength,+ final ByteOrder ordering,+ final HeapIndexFactory<I> indexFactory) throws IOException {+ super(heapFile, keylength, ordering, indexFactory);}/**@@ -118,6 +132,7 @@ public class HeapModifier extends HeapReader implements BLOB {ConcurrentLog.severe("KELONDRO", "BLOBHeap: " + this.heapFile.getName() + ": too long size " + size + " in record at " + seek);throw new IOException("BLOBHeap: " + this.heapFile.getName() + ": too long size " + size + " in record at " + seek);}+ prepareIndexForHeapMutation();super.deleteFingerprint();// add entry to free array@@ -236,18 +251,16 @@ public class HeapModifier extends HeapReader implements BLOB {}}- @Override- public void insert(byte[] key, byte[] b) throws IOException {- throw new UnsupportedOperationException("put is not supported in BLOBHeapModifier");- }-- @Override- public int replace(byte[] key, final Rewriter rewriter) throws IOException {- throw new UnsupportedOperationException();- }-- @Override- public int reduce(byte[] key, final Reducer reducer) throws IOException, SpaceExceededException {+ /**+ * Rewrite an existing BLOB in place without changing its index entry.+ *+ * @param key the primary key+ * @param rewriter produces content no larger than the existing content+ * @return the number of bytes removed from the BLOB+ * @throws IOException when the rewritten content cannot be stored in place+ * @throws SpaceExceededException when the rewriter cannot allocate its result+ */+ protected int rewrite(byte[] key, final Rewriter rewriter) throws IOException, SpaceExceededException {key = normalizeKey(key);assert key.length == this.keylength;@@ -271,6 +284,7 @@ public class HeapModifier extends HeapReader implements BLOB {if (MemoryControl.available() < len) {if (!MemoryControl.request(len, true)) return 0; // not enough memory available for this blob}+ prepareIndexForHeapMutation();super.deleteFingerprint();// read the key@@ -283,7 +297,7 @@ public class HeapModifier extends HeapReader implements BLOB {this.file.readFully(blob, 0, blob.length);// rewrite the entry- blob = reducer.rewrite(blob);+ blob = rewriter.rewrite(blob);int reduction = len - blob.length;if (reduction == 0) {// even if the reduction is zero then it is still be possible that the record has been changed@@ -320,4 +334,9 @@ public class HeapModifier extends HeapReader implements BLOB {}}+ @Override+ public int reduce(final byte[] key, final Reducer reducer) throws IOException, SpaceExceededException {+ return rewrite(key, reducer);+ }+}diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.javaindex 1740e6e29..e1ad337e4 100644--- a/source/net/yacy/kelondro/blob/HeapReader.java+++ b/source/net/yacy/kelondro/blob/HeapReader.java@@ -29,6 +29,8 @@ import java.io.DataInputStream;import java.io.File;import java.io.FileInputStream;import java.io.IOException;+import java.io.UncheckedIOException;+import java.nio.file.Files;import java.util.Arrays;import java.util.Date;import java.util.Iterator;@@ -43,10 +45,12 @@ import net.yacy.cora.order.CloneableIterator;import net.yacy.cora.order.Digest;import net.yacy.cora.order.NaturalOrder;import net.yacy.cora.storage.HandleMap;+import net.yacy.cora.storage.ImmutableHandleMap;import net.yacy.cora.util.ConcurrentLog;import net.yacy.cora.util.LookAheadIterator;import net.yacy.cora.util.SpaceExceededException;import net.yacy.kelondro.index.RowHandleMap;+import net.yacy.kelondro.index.SSTableHandleMap;import net.yacy.kelondro.io.CachedFileWriter;import net.yacy.kelondro.io.Writer;import net.yacy.kelondro.util.FileUtils;@@ -54,12 +58,13 @@ import net.yacy.kelondro.util.MemoryControl;import net.yacy.kelondro.util.RotateIterator;-public class HeapReader {-+public abstract class HeapReader<I extends ImmutableHandleMap> {//public final static long keepFreeMem = 20 * 1024 * 1024;private final static ConcurrentLog log = new ConcurrentLog("KELONDRO");+ private final HeapIndexFactory<I> indexFactory;+// input valuesprotected int keylength; // the length of the primary keyprotected File heapFile; // the file of the heap@@ -67,64 +72,320 @@ public class HeapReader {// computed valuesprotected Writer file; // a random access to the file- protected HandleMap index; // key/seek relation for used records+ protected I index; // key/seek relation for used recordsprotected Gap free; // set of {seek, size} pairs denoting space and position of free recordsprivate File fingerprintFileIdx, fingerprintFileGap; // files with dumped indexes. Will be deleted if file is writtenprivate Date closeDate; // records a time when the file was closed; used for debugging- public HeapReader(++ interface HeapIndexFactory<I extends ImmutableHandleMap> {+ I load(+ int keylength,+ ByteOrder ordering,+ File dump)+ throws IOException, SpaceExceededException;++ HeapIndexBuilder<I> newBuilder(+ File heapFile,+ int keylength,+ ByteOrder ordering,+ int expectedspace) throws IOException;++ void prepareForHeapMutation(I index) throws IOException;+ }++ interface HeapIndexBuilder<I extends ImmutableHandleMap> extends AutoCloseable {+ void consume(byte[] key, long offset) throws IOException;++ I finish() throws IOException;++ @Override+ void close() throws IOException;+ }++ @FunctionalInterface+ interface HeapFileFactory {+ Writer open(File heapFile) throws IOException;+ }++ static HeapIndexFactory<HandleMap> mutable() {+ return new HeapIndexFactory<HandleMap>() {+ @Override+ public HandleMap load(+ final int keylength,+ final ByteOrder ordering,+ final File dump)+ throws IOException, SpaceExceededException {+ return new RowHandleMap(keylength, ordering, 8, dump);+ }++ @Override+ public HeapIndexBuilder<HandleMap> newBuilder(+ final File heapFile,+ final int keylength,+ final ByteOrder ordering,+ final int expectedspace) {+ return new HeapIndexBuilder<HandleMap>() {+ private RowHandleMap.initDataConsumer building =+ RowHandleMap.asynchronusInitializer(+ heapFile.getName() + ".initializer",+ keylength, ordering, 8, expectedspace);+ private boolean finishRequested = false;++ @Override+ public void consume(final byte[] key, final long offset) {+ if (this.building == null) throw new IllegalStateException("index builder is closed");+ this.building.consume(key, offset);+ }++ @Override+ public HandleMap finish() throws IOException {+ if (this.building == null) throw new IllegalStateException("index builder is closed");+ requestFinish();+ final RowHandleMap result = awaitResult();+ this.building = null;+ return result;+ }++ @Override+ public void close() throws IOException {+ if (this.building == null) return;+ requestFinish();+ try {+ awaitResultForClose();+ } finally {+ this.building.close();+ this.building = null;+ }+ }++ private void requestFinish() {+ if (this.finishRequested) return;+ this.building.finish();+ this.finishRequested = true;+ }++ private RowHandleMap awaitResult() throws IOException {+ try {+ return this.building.result();+ } catch (final InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IOException("Interrupted while rebuilding mutable heap index", e);+ } catch (final ExecutionException e) {+ throw new IOException("Cannot rebuild mutable heap index", e.getCause());+ }+ }++ private RowHandleMap awaitResultForClose() throws IOException {+ boolean interrupted = false;+ try {+ while (true) {+ try {+ return this.building.result();+ } catch (final InterruptedException e) {+ interrupted = true;+ } catch (final ExecutionException e) {+ throw new IOException("Cannot rebuild mutable heap index", e.getCause());+ }+ }+ } finally {+ if (interrupted) Thread.currentThread().interrupt();+ }+ }+ };+ }++ @Override+ public void prepareForHeapMutation(final HandleMap index) {+ /* RowHandleMap has no mapped fingerprint backing to detach. */+ }+ };+ }++ /**+ * Index strategy for heaps which never add or replace keys.+ *+ * <p>The concrete generic type is intentional: this is not merely a factory+ * returning some {@link ImmutableHandleMap}. Both loading a fingerprint and+ * rebuilding from the heap are guaranteed to produce an+ * {@link SSTableHandleMap}. A future accidental RowHandleMap fallback would+ * therefore fail at compile time instead of silently restoring the RAM index.</p>+ */+ static HeapIndexFactory<SSTableHandleMap> immutable() {+ return new HeapIndexFactory<SSTableHandleMap>() {+ @Override+ public SSTableHandleMap load(+ final int keylength,+ final ByteOrder ordering,+ final File dump)+ throws IOException, SpaceExceededException {+ /*+ * Fingerprint dumps are published snapshots. SSTable removals write+ * negative offsets, so the dump must never be mapped writable itself.+ * openWorkingCopy() maps it read-only and promotes it to an owned+ * temporary table only when the heap or index is first mutated.+ */+ return SSTableHandleMap.openWorkingCopy(+ keylength, ordering, 8, dump);+ }++ @Override+ public HeapIndexBuilder<SSTableHandleMap> newBuilder(+ final File heapFile,+ final int keylength,+ final ByteOrder ordering,+ final int expectedspace) throws IOException {+ final SSTableHandleMap.Builder builder = new SSTableHandleMap.Builder(+ keylength, ordering, 8, heapFile.getParentFile(), heapFile.getName());+ return new HeapIndexBuilder<SSTableHandleMap>() {+ @Override+ public void consume(final byte[] key, final long offset) throws IOException {+ builder.consume(key, offset);+ }++ @Override+ public SSTableHandleMap finish() throws IOException {+ return builder.finish();+ }++ @Override+ public void close() throws IOException {+ builder.close();+ }+ };+ }++ @Override+ public void prepareForHeapMutation(final SSTableHandleMap index)+ throws IOException {+ index.prepareForMutation();+ }+ };+ }++ protected HeapReader(final File heapFile,final int keylength,- final ByteOrder ordering) throws IOException {+ final ByteOrder ordering,+ final HeapIndexFactory<I> indexFactory) throws IOException {+ this(heapFile, keylength, ordering, indexFactory, CachedFileWriter::new);+ }++ /** Constructor variant for an alternate heap-file ownership implementation. */+ protected HeapReader(+ final File heapFile,+ final int keylength,+ final ByteOrder ordering,+ final HeapIndexFactory<I> indexFactory,+ final HeapFileFactory heapFileFactory) throws IOException {+ if (indexFactory == null) throw new IllegalArgumentException("indexFactory must not be null");+ if (heapFileFactory == null) throw new IllegalArgumentException("heapFileFactory must not be null");+ this.indexFactory = indexFactory;this.ordering = ordering;this.heapFile = heapFile;this.keylength = keylength;this.index = null; // will be created as result of initialization processthis.free = null; // will be initialized later depending on existing idx/gap filethis.heapFile.getParentFile().mkdirs();- this.file = new CachedFileWriter(this.heapFile);+ this.file = heapFileFactory.open(this.heapFile);this.closeDate = null;- // read or initialize the index- this.fingerprintFileIdx = null;- this.fingerprintFileGap = null;- if (initIndexReadDump()) {- // verify that everything worked just fine- // pick some elements of the index- Iterator<byte[]> i = this.index.keys(true, null);- int c = 3;- byte[] b, b1 = new byte[this.keylength];- long pos;- boolean ok = true;- while (i.hasNext() && c-- > 0) {- b = i.next();- pos = this.index.get(b);- this.file.seek(pos + 4);- this.file.readFully(b1, 0, b1.length);- if (!this.ordering.equal(b, b1)) {- ok = false;- break;+ try {+ // read or initialize the index+ this.fingerprintFileIdx = null;+ this.fingerprintFileGap = null;+ if (initIndexReadDump()) {+ if (!verifyLoadedIndex()) {+ log.warn("HeapReader: verification of idx file for " + heapFile.toString() + " failed, re-building index");+ /*+ * Close the working index before invalidating its source fingerprint.+ * This matters for mapped files on Windows and also guarantees that a+ * rejected dump cannot leave a temporary SSTable behind.+ */+ discardIndexState();+ deleteFingerprint();+ initIndexReadFromHeap();+ } else {+ log.info("HeapReader: using a dump of the index of " + heapFile.toString() + ".");}- }- if (!ok) {- log.warn("HeapReader: verification of idx file for " + heapFile.toString() + " failed, re-building index");- initIndexReadFromHeap();} else {- log.info("HeapReader: using a dump of the index of " + heapFile.toString() + ".");+ // if we did not have a dump, create a new index+ initIndexReadFromHeap();}- } else {- // if we did not have a dump, create a new index- initIndexReadFromHeap();++ // merge gaps that follow directly+ mergeFreeEntries();++ // after the initial initialization of the heap, we close the file again+ // to make more room to file pointers which may run out if the number+ // of file descriptors is too low and the number of files is too high+ this.file.close();+ // the file will be opened again automatically when the next access to it comes.+ } catch (final IOException | RuntimeException | Error failure) {+ cleanupFailedInitialization(failure);+ throw failure;}+ }- // merge gaps that follow directly- mergeFreeEntries();+ /** Release every resource acquired by a constructor which cannot return. */+ private void cleanupFailedInitialization(final Throwable failure) {+ if (this.index != null) {+ try {+ this.index.close();+ } catch (final Throwable cleanupFailure) {+ failure.addSuppressed(cleanupFailure);+ } finally {+ this.index = null;+ }+ }+ if (this.free != null) {+ try {+ this.free.clear();+ } catch (final Throwable cleanupFailure) {+ failure.addSuppressed(cleanupFailure);+ } finally {+ this.free = null;+ }+ }+ if (this.file != null) {+ try {+ this.file.close();+ } catch (final Throwable cleanupFailure) {+ failure.addSuppressed(cleanupFailure);+ } finally {+ this.file = null;+ }+ }+ this.closeDate = new Date();+ }- // after the initial initialization of the heap, we close the file again- // to make more room to file pointers which may run out if the number- // of file descriptors is too low and the number of files is too high- this.file.close();- // the file will be opened again automatically when the next access to it comes.+ /**+ * Verify a small sample before the loaded fingerprint becomes normal runtime state.+ *+ * <p>The index and gap dump form a cache, while the heap file remains the source of+ * truth. A key mismatch, an invalid offset, or a short heap read therefore rejects+ * the complete cached generation and lets the constructor rebuild it from the heap.+ * The iterator is closed here so this validation phase owns all resources it opens.</p>+ */+ private boolean verifyLoadedIndex() {+ final CloneableIterator<byte[]> keys = this.index.keys(true, null);+ try {+ int remaining = 3;+ final byte[] storedKey = new byte[this.keylength];+ while (keys.hasNext() && remaining-- > 0) {+ final byte[] indexedKey = keys.next();+ final long position = this.index.get(indexedKey);+ this.file.seek(position + 4);+ this.file.readFully(storedKey, 0, storedKey.length);+ if (!this.ordering.equal(indexedKey, storedKey)) return false;+ }+ return true;+ } catch (final IOException | RuntimeException e) {+ log.warn("HeapReader: cannot verify loaded index for " + this.heapFile, e);+ return false;+ } finally {+ keys.close();+ }}public long mem() {@@ -135,6 +396,11 @@ public class HeapReader {this.index.optimize();}+ /** Detach a mapped fingerprint before changing the heap generation. */+ protected final void prepareIndexForHeapMutation() throws IOException {+ this.indexFactory.prepareForHeapMutation(this.index);+ }+protected byte[] normalizeKey(byte[] key) {// check size of key: zero-filled keys are only possible of the ordering is// an instance of the natural ordering. Base64-orderings cannot use zeros in keys.@@ -170,39 +436,71 @@ public class HeapReader {if (!this.fingerprintFileGap.exists()) this.fingerprintFileGap = new File(this.fingerprintFileGap.getAbsolutePath() + ".gz");if (!this.fingerprintFileIdx.exists() || !this.fingerprintFileGap.exists()) {deleteAllFingerprints(this.heapFile, this.fingerprintFileIdx.getName(), this.fingerprintFileGap.getName());+ /* An incomplete pair must not be mistaken for a publishable snapshot. */+ deleteFingerprint();return false;}- // there is an index and a gap file:- // read the index file:+ /*+ * Load both parts transactionally. In the immutable mode indexFactory.load()+ * maps the published fingerprint read-only; it creates a working copy only+ * on a later mutation. The loaded index must be closed immediately when+ * loading the gap file or validating the pair fails; assigning fields early+ * would leak that mapping when the heap scan starts its replacement builder.+ */+ I loadedIndex = null;try {- this.index = new RowHandleMap(this.keylength, this.ordering, 8, this.fingerprintFileIdx);+ loadedIndex = this.indexFactory.load(+ this.keylength, this.ordering, this.fingerprintFileIdx);} catch (final IOException e) {ConcurrentLog.logException(e);+ deleteFingerprint();return false;} catch (final SpaceExceededException e) {ConcurrentLog.logException(e);+ deleteFingerprint();return false;}// check saturation- if (this.index instanceof RowHandleMap) {- int[] saturation = ((RowHandleMap) this.index).saturation(); // {<the maximum length of consecutive equal-beginning bytes in the key>, <the minimum number of leading zeros in the second column>}- log.info("HeapReader: saturation of " + this.fingerprintFileIdx.getName() + ": keylength = " + saturation[0] + ", vallength = " + saturation[1] + ", size = " + this.index.size() +- ", maximum saving for index-compression = " + (saturation[0] * this.index.size() / 1024 / 1024) + " MB" +- ", exact saving for value-compression = " + (saturation[1] * this.index.size() / 1024 / 1024) + " MB");+ if (loadedIndex instanceof RowHandleMap) {+ final int[] saturation = ((RowHandleMap) loadedIndex).saturation(); // {<the maximum length of consecutive equal-beginning bytes in the key>, <the minimum number of leading zeros in the second column>}+ log.info("HeapReader: saturation of " + this.fingerprintFileIdx.getName() + ": keylength = " + saturation[0] + ", vallength = " + saturation[1] + ", size = " + loadedIndex.size() ++ ", maximum saving for index-compression = " + (saturation[0] * loadedIndex.size() / 1024 / 1024) + " MB" ++ ", exact saving for value-compression = " + (saturation[1] * loadedIndex.size() / 1024 / 1024) + " MB");}- // read the gap file:+ final Gap loadedFree;try {- this.free = new Gap(this.fingerprintFileGap);+ loadedFree = new Gap(this.fingerprintFileGap);} catch (final IOException e) {- ConcurrentLog.logException(e);+ loadedIndex.close();+ ConcurrentLog.logException(e);+ deleteFingerprint();return false;}- // everything is fine now- return !this.index.isEmpty();+ if (loadedIndex.isEmpty() && this.heapFile.length() > 0) {+ loadedIndex.close();+ loadedFree.clear();+ deleteFingerprint();+ return false;+ }++ this.index = loadedIndex;+ this.free = loadedFree;+ return true;+ }++ private void discardIndexState() {+ if (this.index != null) {+ this.index.close();+ this.index = null;+ }+ if (this.free != null) {+ this.free.clear();+ this.free = null;+ }}/**@@ -252,65 +550,60 @@ public class HeapReader {log.info("HeapReader: generating index for " + this.heapFile.toString() + ", " + (this.file.length() / 1024 / 1024) + " MB. Please wait.");this.free = new Gap();- RowHandleMap.initDataConsumer indexready = RowHandleMap.asynchronusInitializer(this.name() + ".initializer", this.keylength, this.ordering, 8, Math.max(10, (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024))));- byte[] key = new byte[this.keylength];- int reclen;- long seek = 0;- if (this.file.length() > 0) {- loop: while (true) { // don't test available() here because this does not work for files > 2GB+ final int expectedspace = Math.max(10,+ (int) (Runtime.getRuntime().freeMemory() / (10 * 1024 * 1024)));+ try (HeapIndexBuilder<I> indexBuilder = this.indexFactory.newBuilder(+ this.heapFile, this.keylength, this.ordering, expectedspace)) {+ byte[] key = new byte[this.keylength];+ int reclen;+ long seek = 0;+ if (this.file.length() > 0) {+ loop: while (true) { // don't test available() here because this does not work for files > 2GB- try {- // go to seek position- this.file.seek(seek);-- // read length of the following record without the length of the record size bytes- reclen = this.file.readInt();- //assert reclen > 0 : " reclen == 0 at seek pos " + seek;- if (reclen == 0) {- // very bad file inconsistency- log.severe("HeapReader: reclen == 0 at seek pos " + seek + " in file " + this.heapFile);- this.file.setLength(seek); // delete everything else at the remaining of the file :-(- break loop;- }+ try {+ // go to seek position+ this.file.seek(seek);++ // read length of the following record without the length of the record size bytes+ reclen = this.file.readInt();+ //assert reclen > 0 : " reclen == 0 at seek pos " + seek;+ if (reclen == 0) {+ // very bad file inconsistency+ log.severe("HeapReader: reclen == 0 at seek pos " + seek + " in file " + this.heapFile);+ this.file.setLength(seek); // delete everything else at the remaining of the file :-(+ break loop;+ }- // read key- this.file.readFully(key, 0, key.length);+ // read key+ this.file.readFully(key, 0, key.length);- } catch (final IOException e) {- // EOF reached- break loop; // terminate loop- }+ } catch (final IOException e) {+ // EOF reached+ break loop; // terminate loop+ }- // check if this record is empty- if (key == null || key[0] == 0) {- // it is an empty record, store to free list- if (reclen > 0) this.free.put(seek, reclen);- } else {- if (this.ordering.wellformed(key)) {- indexready.consume(key, seek);- key = new byte[this.keylength];+ // check if this record is empty+ if (key == null || key[0] == 0) {+ // it is an empty record, store to free list+ if (reclen > 0) this.free.put(seek, reclen);} else {- // free the lost space- this.free.put(seek, reclen);- this.file.seek(seek + 4);- Arrays.fill(key, (byte) 0);- this.file.write(key); // mark the place as empty record- log.warn("HeapReader: BLOB " + this.heapFile.getName() + ": skiped not wellformed key " + UTF8.String(key) + " at seek pos " + seek);+ if (this.ordering.wellformed(key)) {+ indexBuilder.consume(key, seek);+ key = new byte[this.keylength];+ } else {+ // free the lost space+ this.free.put(seek, reclen);+ this.file.seek(seek + 4);+ Arrays.fill(key, (byte) 0);+ this.file.write(key); // mark the place as empty record+ log.warn("HeapReader: BLOB " + this.heapFile.getName() + ": skiped not wellformed key " + UTF8.String(key) + " at seek pos " + seek);+ }}+ // new seek position+ seek += 4L + reclen;}- // new seek position- seek += 4L + reclen;- }- }- indexready.finish();-- // finish the index generation- try {- this.index = indexready.result();- } catch (final InterruptedException e) {- ConcurrentLog.logException(e);- } catch (final ExecutionException e) {- ConcurrentLog.logException(e);+ }+ this.index = indexBuilder.finish();}log.info("HeapReader: finished index generation for " + this.heapFile.toString() + ", " + this.index.size() + " entries, " + this.free.size() + " gaps.");}@@ -328,6 +621,7 @@ public class HeapReader {//System.out.println("*** DEBUG BLOB: free-seek = " + nextFree.seek + ", size = " + nextFree.size);// check if they follow directlyif (lastFree.getKey() + lastFree.getValue() + 4 == nextFree.getKey()) {+ if (merged == 0) prepareIndexForHeapMutation();// merge those recordsthis.file.seek(lastFree.getKey());lastFree.setValue(lastFree.getValue() + nextFree.getValue() + 4); // this updates also the free map@@ -502,6 +796,7 @@ public class HeapReader {log.severe("HeapReader: file " + this.file.file() + " corrupted at " + pos + ": negative len. len = " + len + ", pk.len = " + this.keylength);// to get lazy over that problem (who wants to tell the user to stop operation and delete the file???) we work on like the entry does not existthis.index.remove(key);+ deleteFingerprint();return null;}long memr = len + this.keylength + 64;@@ -523,6 +818,7 @@ public class HeapReader {// this is a severe operation, it should never happen.// remove entry from index because keeping that element in the index would not make sensethis.index.remove(key);+ deleteFingerprint();// nothing to returnreturn null;// but if the process ends in this state, it would completely fail@@ -599,67 +895,119 @@ public class HeapReader {/*** close the BLOB table+ * @throws UncheckedIOException when the heap cannot be closed or its complete+ * fingerprint pair cannot be published*/public void close(boolean writeIDX) {- if (this.index == null) return;- synchronized (this.index) {+ final I closingIndex = this.index;+ if (closingIndex == null) return;+ Throwable failure = null;+ synchronized (closingIndex) {try {- if (this.file != null)- try {- this.file.close();- } catch (final IOException e) {- ConcurrentLog.logException(e);- }- this.file = null;- if (writeIDX && this.index != null && this.free != null && (this.index.size() > 3 || this.free.size() > 3)) {- // now we can create a dump of the index and the gap information- // to speed up the next start+ if (this.file != null) this.file.close();+ this.file = null;+ if (writeIDX && this.free != null+ && (closingIndex.size() > 3 || this.free.size() > 3)) {+ publishFingerprintGeneration(+ this.heapFile, closingIndex, this.free,+ this.fingerprintFileIdx, this.fingerprintFileGap,+ "HeapReader");+ }+ } catch (final Throwable e) {+ failure = e;+ } finally {+ this.file = null;+ if (this.free != null) this.free.clear();+ this.free = null;try {- String fingerprint = fingerprintFileHash(this.heapFile);- if (fingerprint == null) {- log.severe("HeapReader: cannot write a dump for " + this.heapFile.getName()+ ": fingerprint is null");- } else {- File newFingerprintFileGap = HeapWriter.fingerprintGapFile(this.heapFile, fingerprint);- if (this.fingerprintFileGap != null &&- this.fingerprintFileGap.getName().equals(newFingerprintFileGap.getName()) &&- this.fingerprintFileGap.exists()) {- log.info("HeapReader: using existing gap dump instead of writing a new one: " + this.fingerprintFileGap.getName());- } else {- long start = System.currentTimeMillis();- this.free.dump(newFingerprintFileGap);- log.info("HeapReader: wrote a dump for the " + this.free.size() + " gap entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");- }- }- this.free.clear();- this.free = null;- if (fingerprint != null) {- File newFingerprintFileIdx = HeapWriter.fingerprintIndexFile(this.heapFile, fingerprint);- if (this.fingerprintFileIdx != null &&- this.fingerprintFileIdx.getName().equals(newFingerprintFileIdx.getName()) &&- this.fingerprintFileIdx.exists()) {- log.info("HeapReader: using existing idx dump instead of writing a new one: " + this.fingerprintFileIdx.getName());- } else {- long start = System.currentTimeMillis();- this.index.dump(newFingerprintFileIdx);- log.info("HeapReader: wrote a dump for the " + this.index.size() + " index entries of " + this.heapFile.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");- }- }- this.index.close();- this.index = null;- } catch (final IOException e) {- ConcurrentLog.logException(e);+ closingIndex.close();+ } catch (final Throwable closeFailure) {+ if (failure == null) failure = closeFailure;+ else failure.addSuppressed(closeFailure);}+ if (this.index == closingIndex) this.index = null;+ this.closeDate = new Date();}- if (this.free != null) this.free.clear();- this.free = null;- if (this.index != null) this.index.close();- this.index = null;- this.closeDate = new Date();- } catch (Throwable e) {ConcurrentLog.logException(e);}log.info("HeapReader: close HeapFile " + this.heapFile.getName());log.fine("trace: " + ConcurrentLog.stackTrace());}++ if (failure instanceof IOException) {+ final IOException ioFailure = (IOException) failure;+ log.severe("HeapReader: cannot close and publish fingerprint for "+ + this.heapFile, ioFailure);+ throw new UncheckedIOException(+ "Cannot close and publish fingerprint for " + this.heapFile,+ ioFailure);+ }+ if (failure instanceof RuntimeException) throw (RuntimeException) failure;+ if (failure instanceof Error) throw (Error) failure;+ if (failure != null) {+ throw new IllegalStateException("Cannot close heap " + this.heapFile, failure);+ }+ }++ /**+ * Publish one coherent cache generation. The index is installed first and the+ * gap file last; the latter acts as the commit marker because readers require+ * both files. A reported failure removes every part installed by this attempt.+ */+ static void publishFingerprintGeneration(+ final File heapFile, final ImmutableHandleMap index, final Gap free,+ final File reusableIndex, final File reusableGap,+ final String publisher) throws IOException {+ final String fingerprint = fingerprintFileHash(heapFile);+ if (fingerprint == null) {+ throw new IOException(publisher + ": cannot publish fingerprint for "+ + heapFile + ": fingerprint is null");+ }++ final File targetIndex = HeapWriter.fingerprintIndexFile(heapFile, fingerprint);+ final File targetGap = HeapWriter.fingerprintGapFile(heapFile, fingerprint);+ final boolean reusePair = sameFingerprintFile(reusableIndex, targetIndex)+ && sameFingerprintFile(reusableGap, targetGap);+ if (reusePair) {+ log.info(publisher + ": using existing fingerprint pair for "+ + heapFile.getName());+ return;+ }++ final long start = System.currentTimeMillis();+ boolean indexPublished = false;+ boolean gapPublished = false;+ try {+ index.dump(targetIndex);+ indexPublished = true;+ free.dump(targetGap);+ gapPublished = true;+ log.info(publisher + ": published a fingerprint with " + index.size()+ + " index entries and " + free.size() + " gap entries for "+ + heapFile.getName() + " in "+ + (System.currentTimeMillis() - start) + " milliseconds.");+ } catch (final IOException | RuntimeException e) {+ final IOException failure = new IOException(+ publisher + ": cannot publish complete fingerprint pair for "+ + heapFile, e);+ if (gapPublished) deleteFailedPublish(targetGap, failure);+ if (indexPublished) deleteFailedPublish(targetIndex, failure);+ throw failure;+ }+ }++ private static boolean sameFingerprintFile(+ final File reusable, final File target) {+ return reusable != null && reusable.exists()+ && reusable.getAbsoluteFile().equals(target.getAbsoluteFile());+ }++ private static void deleteFailedPublish(+ final File file, final IOException failure) {+ try {+ Files.deleteIfExists(file.toPath());+ } catch (final IOException cleanupFailure) {+ failure.addSuppressed(cleanupFailure);+ }}public synchronized void close() {diff --git a/source/net/yacy/kelondro/blob/HeapWriter.java b/source/net/yacy/kelondro/blob/HeapWriter.javaindex 81da990fd..82b9dd378 100644--- a/source/net/yacy/kelondro/blob/HeapWriter.java+++ b/source/net/yacy/kelondro/blob/HeapWriter.java@@ -152,25 +152,26 @@ public final class HeapWriter {if (this.heapFileTMP.exists()) throw new IOException("HeapWriter: renaming to " + this.heapFileTMP.toString() + " failed: file does not exist");// generate index and gap files- if (writeIDX && this.index.size() > 3) {- // now we can create a dump of the index and the gap information- // to speed up the next start- long start = System.currentTimeMillis();- String fingerprint = HeapReader.fingerprintFileHash(this.heapFileREADY);- if (fingerprint == null) {- log.severe("HeapWriter: cannot write a dump for " + this.heapFileREADY.getName()+ ": fingerprint is null");- } else {- new Gap().dump(fingerprintGapFile(this.heapFileREADY, fingerprint));- this.index.dump(fingerprintIndexFile(this.heapFileREADY, fingerprint));- log.info("HeapWriter: wrote a dump for the " + this.index.size() + " index entries of " + this.heapFileREADY.getName()+ " in " + (System.currentTimeMillis() - start) + " milliseconds.");+ IOException publishFailure = null;+ try {+ if (writeIDX && this.index.size() > 3) {+ HeapReader.publishFingerprintGeneration(+ this.heapFileREADY, this.index, new Gap(),+ null, null, "HeapWriter");+ }+ } catch (final IOException e) {+ publishFailure = e;+ } finally {+ try {+ this.index.close();+ } catch (final RuntimeException closeFailure) {+ if (publishFailure == null) throw closeFailure;+ publishFailure.addSuppressed(closeFailure);+ } finally {+ this.index = null;}- this.index.close();- this.index = null;- } else {- // this is small.. just free resources, do not write index- this.index.close();- this.index = null;}+ if (publishFailure != null) throw publishFailure;}public static void delete(File f) {diff --git a/source/net/yacy/kelondro/blob/ImmutableBLOB.java b/source/net/yacy/kelondro/blob/ImmutableBLOB.javanew file mode 100644index 000000000..ee78728fe--- /dev/null+++ b/source/net/yacy/kelondro/blob/ImmutableBLOB.java@@ -0,0 +1,180 @@+// ImmutableBLOB.java+// (C) 2008 by Michael Peter Christen; [email protected], Frankfurt a. M., Germany+// first published 08.06.2008 on http://yacy.net+//+// This is a part of YaCy, a peer-to-peer based web search engine+//+// LICENSE+//+// This program is free software; you can redistribute it and/or modify+// it under the terms of the GNU General Public License as published by+// the Free Software Foundation; either version 2 of the License, or+// (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU General Public License for more details.+//+// You should have received a copy of the GNU General Public License+// along with this program; if not, write to the Free Software+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA++package net.yacy.kelondro.blob;++import java.io.IOException;+import java.io.UncheckedIOException;++import net.yacy.cora.order.ByteOrder;+import net.yacy.cora.order.CloneableIterator;+import net.yacy.cora.util.SpaceExceededException;+++public interface ImmutableBLOB {++ /**+ * return a name of the BLOB; can be the file name+ * @return+ */+ public String name();++ /**+ * ask for the length of the primary key+ * @return the length of the key+ */+ public int keylength();++ /**+ * return the underlying odering+ * @return+ */+ public ByteOrder ordering();++ /**+ * clears the content of the database+ * @throws IOException+ */+ public void clear() throws IOException;++ /**+ * trim the index of the database: this releases memory not currently used+ * @throws IOException+ */+ public void optimize();++ /**+ * calculate the memory in RAM that the BLOB occupies+ * @return number of bytes that is used+ */+ public long mem();++ /**+ * ask for the number of entries+ * @return the number of entries in the table+ */+ public int size();++ /**+ * ask if the BLOB is empty+ * @return true iff size() == 0+ */+ public boolean isEmpty();++ /**+ * iterator over all keys+ * @param up+ * @param rotating+ * @return+ * @throws IOException+ */+ public CloneableIterator<byte[]> keys(boolean up, boolean rotating) throws IOException;++ /**+ * iterate over all keys+ * @param up+ * @param firstKey+ * @return+ * @throws IOException+ */+ public CloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException;++ /**+ * check if a specific key is in the database+ * @param key the primary key+ * @return+ * @throws IOException+ */+ public boolean containsKey(byte[] key);++ /**+ * retrieve the whole BLOB from the table+ * @param key the primary key+ * @return+ * @throws IOException+ */+ public byte[] get(byte[] key) throws IOException, SpaceExceededException;+ public byte[] get(Object key);++ /**+ * retrieve the size of the database+ * @param+ * @return the size of the database in bytes+ * @throws IOException+ */+ public long length() throws IOException;++ /**+ * retrieve the size of the BLOB+ * @param key+ * @return the size of the BLOB or -1 if the BLOB does not exist+ * @throws IOException+ */+ public long length(byte[] key) throws IOException;++ /**+ * remove a BLOB+ * @param key the primary key+ * @throws IOException+ */+ public void delete(byte[] key) throws IOException;++ /**+ * a reduce method is the same as the replace. A replace subsumes a reduce method. A reduce method may be more simple.+ * @param key the primary key+ * @param reducer+ * @return the number of bytes that the rewriter reduced the BLOB+ * @throws IOException+ * @throws SpaceExceededException+ */+ public int reduce(byte[] key, Reducer reducer) throws IOException, SpaceExceededException;++ public interface Rewriter {+ /**+ * a rewrite method that is used in the replace functionality of a BLOB+ * the result of such a rewrite must be always smaller or equal in size+ * of the input+ * @param b+ * @return an array that is equal or smaller in size than b+ * @throws SpaceExceededException+ */+ public byte[] rewrite(byte[] b) throws SpaceExceededException;+ }++ public interface Reducer extends Rewriter {+ /**+ * A Reducer is a rewriter that reduced the content. There are no additional methods in this interface.+ * The interface shall be used in replacement of a Rewriter to simply state the fact that the rewritement+ * also reduces the content of a BLOB entry or may also keep the size the same;+ */+ }++ /**+ * Close the BLOB table and optionally publish its index fingerprint.+ *+ * @throws UncheckedIOException when closing storage or publishing the complete+ * fingerprint pair fails; the signature remains unchecked for compatibility+ * with the existing BLOB lifecycle+ */+ public void close(boolean writeIDX);++}diff --git a/source/net/yacy/kelondro/index/RowHandleMap.java b/source/net/yacy/kelondro/index/RowHandleMap.javaindex f5a8f4b1d..dbeda5479 100644--- a/source/net/yacy/kelondro/index/RowHandleMap.java+++ b/source/net/yacy/kelondro/index/RowHandleMap.java@@ -32,6 +32,9 @@ import java.io.FileOutputStream;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;+import java.nio.file.AtomicMoveNotSupportedException;+import java.nio.file.Files;+import java.nio.file.StandardCopyOption;import java.util.AbstractMap;import java.util.ArrayList;import java.util.Iterator;@@ -52,7 +55,6 @@ import java.util.zip.GZIPOutputStream;import net.yacy.cora.order.ByteOrder;import net.yacy.cora.order.CloneableIterator;import net.yacy.cora.storage.HandleMap;-import net.yacy.cora.util.ConcurrentLog;import net.yacy.cora.util.SpaceExceededException;import net.yacy.kelondro.util.NamePrefixThreadFactory;import net.yacy.kelondro.workflow.WorkflowProcessor;@@ -200,34 +202,48 @@ public final class RowHandleMap implements HandleMap, Iterable<Map.Entry<byte[],final File tmp = new File(file.getParentFile(), file.getName() + ".prt");final Iterator<Row.Entry> i = this.index.rows(true, null);int c = 0;- final FileOutputStream fileStream = new FileOutputStream(tmp);- OutputStream os = null;try {- try {- os = new BufferedOutputStream(fileStream, 4 * 1024 * 1024);- } catch (final OutOfMemoryError e) {- os = fileStream;- }- if (file.getName().endsWith(".gz")) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};- while (i.hasNext()) {- os.write(i.next().bytes());- c++;- }- os.flush();- } finally {- try {- if(os != null) {- os.close();- }- } finally {- if(fileStream != os) {- fileStream.close();- }- }+ final FileOutputStream fileStream = new FileOutputStream(tmp);+ OutputStream os = null;+ try {+ try {+ os = new BufferedOutputStream(fileStream, 4 * 1024 * 1024);+ } catch (final OutOfMemoryError e) {+ os = fileStream;+ }+ if (file.getName().endsWith(".gz")) {+ os = new GZIPOutputStream(os, 65536) {{+ this.def.setLevel(Deflater.BEST_COMPRESSION);+ }};+ }+ while (i.hasNext()) {+ os.write(i.next().bytes());+ c++;+ }+ os.flush();+ } finally {+ try {+ if (os != null) os.close();+ } finally {+ if (fileStream != os) fileStream.close();+ }+ }+ try {+ Files.move(tmp.toPath(), file.toPath(),+ StandardCopyOption.ATOMIC_MOVE,+ StandardCopyOption.REPLACE_EXISTING);+ } catch (final AtomicMoveNotSupportedException e) {+ Files.move(tmp.toPath(), file.toPath(),+ StandardCopyOption.REPLACE_EXISTING);+ }+ } catch (final IOException e) {+ try {+ Files.deleteIfExists(tmp.toPath());+ } catch (final IOException cleanupFailure) {+ e.addSuppressed(cleanupFailure);+ }+ throw e;}- tmp.renameTo(file);- assert file.exists() : file.toString();- assert !tmp.exists() : tmp.toString();return c;}@@ -490,21 +506,17 @@ public final class RowHandleMap implements HandleMap, Iterable<Map.Entry<byte[],}@Override- public final RowHandleMap call() throws IOException {- try {- finishloop: while (true) {- entry c;- try {- while ((c = this.cache.take()) != poisonEntry) {- this.map.putUnique(c.key, c.l);- }- break finishloop;- } catch (final InterruptedException e) {- continue finishloop;+ public final RowHandleMap call() throws SpaceExceededException {+ finishloop: while (true) {+ entry c;+ try {+ while ((c = this.cache.take()) != poisonEntry) {+ this.map.putUnique(c.key, c.l);}+ break finishloop;+ } catch (final InterruptedException e) {+ continue finishloop;}- } catch (final SpaceExceededException e) {- ConcurrentLog.logException(e);}return this.map;}diff --git a/source/net/yacy/kelondro/index/SSTableHandleMap.java b/source/net/yacy/kelondro/index/SSTableHandleMap.javanew file mode 100644index 000000000..fce3b5e72--- /dev/null+++ b/source/net/yacy/kelondro/index/SSTableHandleMap.java@@ -0,0 +1,1420 @@+/*+ * SSTableHandleMap+ * Copyright 2026 by Michael Peter Christen; [email protected], Frankfurt a. M., Germany+ * First published 30.08.2026 on http://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ */++package net.yacy.kelondro.index;++import java.io.BufferedInputStream;+import java.io.BufferedOutputStream;+import java.io.EOFException;+import java.io.File;+import java.io.FileOutputStream;+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.io.UncheckedIOException;+import java.nio.channels.FileChannel;+import java.nio.channels.FileLock;+import java.nio.channels.OverlappingFileLockException;+import java.nio.file.AtomicMoveNotSupportedException;+import java.nio.file.FileVisitResult;+import java.nio.file.Files;+import java.nio.file.LinkOption;+import java.nio.file.NoSuchFileException;+import java.nio.file.Path;+import java.nio.file.SimpleFileVisitor;+import java.nio.file.StandardCopyOption;+import java.nio.file.StandardOpenOption;+import java.nio.file.attribute.BasicFileAttributes;+import java.util.AbstractMap;+import java.util.ArrayList;+import java.util.Comparator;+import java.util.Iterator;+import java.util.List;+import java.util.Map;+import java.util.NoSuchElementException;+import java.util.PriorityQueue;+import java.util.UUID;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.locks.ReentrantReadWriteLock;+import java.util.regex.Matcher;+import java.util.regex.Pattern;+import java.util.zip.Deflater;+import java.util.zip.GZIPInputStream;+import java.util.zip.GZIPOutputStream;++import net.yacy.cora.order.ByteOrder;+import net.yacy.cora.order.CloneableIterator;+import net.yacy.cora.order.NaturalOrder;+import net.yacy.cora.storage.ImmutableHandleMap;+import net.yacy.cora.util.ChunkedBytes;+import net.yacy.cora.util.SpaceExceededException;++/**+ * A file-backed {@link ImmutableHandleMap} over a sorted fixed-width index.+ *+ * <p>The backing file contains {@code keyLength + valueLength} byte records.+ * Keys are sorted with the supplied {@link ByteOrder}; values use the b256+ * encoding used by {@link RowHandleMap}. Lookups binary-search the mapped file,+ * so the number of entries does not determine Java heap consumption.</p>+ *+ * <p>Keys are immutable, but removals replace their non-negative file offset+ * with {@code -1} directly in the table. This keeps deletion state with the+ * index and needs no sidecar file. Adding or changing live values is deliberately+ * not supported. {@link #dump(File)} omits deleted records while preserving the+ * existing RowHandleMap dump format.</p>+ */+public final class SSTableHandleMap implements ImmutableHandleMap, Iterable<Map.Entry<byte[], Long>> {++ private static final int COPY_BUFFER_SIZE = 64 * 1024;+ private static final int DEFAULT_RUN_MEMORY_BYTES = 8 * 1024 * 1024;+ private static final int MAX_MERGE_FAN_IN = 64;+ private static final int ESTIMATED_ARRAY_OVERHEAD = 24;+ private static final byte[] DELETED_OFFSET = NaturalOrder.encodeLong(-1L, Long.BYTES);+ private static final String BUILD_LOCK_SUFFIX = ".sstlock";+ private static final Pattern WORKING_ARTIFACT = Pattern.compile(+ "^sst-work-[0-9a-f]{1,8}-.+\\.sstwork$");+ private static final Pattern BUILD_ARTIFACT = Pattern.compile(+ "^(sst-build-[0-9a-f]{1,8}-[0-9a-f]{32})-.+\\.(?:run|merge|sst)$");+ private static final Pattern BUILD_LOCK_ARTIFACT = Pattern.compile(+ "^sst-build-[0-9a-f]{1,8}-[0-9a-f]{32}\\.sstlock$");+ private static final Pattern LEGACY_BUILD_ARTIFACT = Pattern.compile(+ "^sst-[0-9a-f]{1,8}-.+\\.(?:run|merge|sst)$");++ private final int keyLength;+ private final int valueLength;+ private final int recordLength;+ private final int recordCount;+ private final ByteOrder ordering;+ private final AtomicInteger liveCount;+ private final Path promotionDirectory;+ private final String promotionPrefix;+ private final ReentrantReadWriteLock backingLock;++ /* These fields change together while backingLock's write lock is held. */+ private volatile File tableFile;+ private volatile ChunkedBytes table;+ private boolean writable;+ private boolean deleteOnClose;+ private FileChannel ownershipChannel;+ private FileLock ownershipLock;++ private volatile boolean closed;++ /** Result of one repository-wide temporary-artifact recovery pass. */+ public static final class RecoveryReport {+ private long candidates;+ private long deleted;+ private long inUse;+ private long failures;++ public long candidates() {+ return this.candidates;+ }++ public long deleted() {+ return this.deleted;+ }++ public long inUse() {+ return this.inUse;+ }++ public long failures() {+ return this.failures;+ }+ }++ /**+ * Open a sorted RowHandleMap dump as a Sorted String Table.+ * (where those strings may be represented by binary objects)+ *+ * @param keyLength fixed key width+ * @param ordering ordering used to sort keys in the file+ * @param valueLength fixed b256 value width; offsets require eight bytes+ * @param file uncompressed RowHandleMap dump+ * @throws IOException when the table is invalid+ */+ public SSTableHandleMap(final int keyLength, final ByteOrder ordering,+ final int valueLength, final File file) throws IOException {+ this(keyLength, ordering, valueLength, file,+ false, null, null, true, null, null);+ }++ private SSTableHandleMap(final int keyLength, final ByteOrder ordering,+ final int valueLength, final File file, final boolean deleteOnClose) throws IOException {+ this(keyLength, ordering, valueLength, file,+ deleteOnClose, null, null, true, null, null);+ }++ private SSTableHandleMap(final int keyLength, final ByteOrder ordering,+ final int valueLength, final File file, final boolean deleteOnClose,+ final FileChannel ownershipChannel, final FileLock ownershipLock,+ final boolean writable, final Path promotionDirectory,+ final String promotionPrefix) throws IOException {+ if (keyLength <= 0) throw new IllegalArgumentException("keyLength must be positive");+ if (valueLength != Long.BYTES) {+ throw new IllegalArgumentException("SSTable offsets require valueLength 8");+ }+ if (ordering == null) throw new NullPointerException("ordering");+ if (file == null) throw new NullPointerException("file");+ if (file.getName().endsWith(".gz")) {+ throw new IOException("SSTableHandleMap requires an uncompressed index: " + file);+ }+ if (!file.isFile()) throw new IOException("SSTable index does not exist: " + file);++ this.keyLength = keyLength;+ this.valueLength = valueLength;+ this.recordLength = Math.addExact(keyLength, valueLength);+ final long fileLength = file.length();+ if (fileLength % this.recordLength != 0) {+ throw new IOException("SSTable index length " + fileLength+ + " is not a multiple of record length " + this.recordLength + ": " + file);+ }+ final long records = fileLength / this.recordLength;+ if (records > Integer.MAX_VALUE) {+ throw new IOException("SSTable contains too many records for HandleMap: " + records);+ }++ this.recordCount = (int) records;+ this.ordering = ordering;+ this.tableFile = file.getAbsoluteFile();+ this.table = mapFile(this.tableFile, writable);+ this.writable = writable;+ this.deleteOnClose = deleteOnClose;+ this.ownershipChannel = ownershipChannel;+ this.ownershipLock = ownershipLock;+ this.promotionDirectory = promotionDirectory;+ this.promotionPrefix = promotionPrefix;+ this.backingLock = new ReentrantReadWriteLock();+ this.closed = false;++ final int live;+ try {+ if (this.table.size() != fileLength) {+ throw new IOException("SSTable changed while it was opened: expected "+ + fileLength + " bytes, mapped " + this.table.size() + ": " + file);+ }+ live = validateRecords();+ } catch (final IOException | RuntimeException | Error failure) {+ try {+ this.table.close();+ } catch (final RuntimeException closeFailure) {+ failure.addSuppressed(closeFailure);+ }+ throw failure;+ }+ this.liveCount = new AtomicInteger(live);+ }++ /**+ * Validate every record before binary-searching the mapped table. Alignment+ * alone is insufficient: an out-of-order or duplicate key would make lookups+ * return false negatives, while a negative value other than the designated+ * {@code -1} tombstone has no defined meaning.+ */+ private int validateRecords() throws IOException {+ final byte[] previousKey = new byte[this.keyLength];+ final byte[] record = new byte[this.recordLength];+ int live = 0;++ for (int ordinal = 0; ordinal < this.recordCount; ordinal++) {+ readRecord(ordinal, record);+ if (ordinal > 0) {+ final int comparison = this.ordering.compare(+ previousKey, 0, record, 0, this.keyLength);+ if (comparison >= 0) {+ final String reason = comparison == 0+ ? "duplicate key" : "keys are out of order";+ throw new IOException("Invalid SSTable at records "+ + (ordinal - 1) + " and " + ordinal + ": " + reason+ + ": " + this.tableFile);+ }+ }++ final long value = NaturalOrder.decodeLong(+ record, this.keyLength, this.valueLength);+ if (value == -1L) {+ /* The only valid negative value is the deletion tombstone. */+ } else if (value < 0L) {+ throw new IOException("Invalid negative SSTable offset " + value+ + " at record " + ordinal + ": " + this.tableFile);+ } else {+ live++;+ }+ System.arraycopy(record, 0, previousKey, 0, this.keyLength);+ }+ return live;+ }++ /**+ * Open an immutable fingerprint snapshot with lazy writable promotion.+ *+ * <p>A fingerprint index belongs to one exact heap generation and is therefore+ * treated as a published, immutable cache artifact. This class nevertheless+ * has to persist removals by writing {@code -1} offsets. Mapping the fingerprint+ * itself writable would mix these two lifecycles and would also make deletion or+ * replacement of a mapped fingerprint unsafe on platforms such as Windows.</p>+ *+ * <p>An uncompressed snapshot is initially mapped read-only without creating a+ * working file. {@link #prepareForMutation()}, {@link #remove(byte[])},+ * {@link #removeone()}, or {@link #clear()} atomically copy the mapped bytes to a+ * private locked working file, switch the active mapping, and unmap the published+ * snapshot. This order also permits HeapReader to invalidate or replace the+ * fingerprint safely on Windows. A gzip snapshot cannot be mapped and is therefore+ * decompressed eagerly into its private working file.</p>+ *+ * @param sourceFingerprint sorted RowHandleMap fingerprint dump, optionally gzip compressed+ * @return an SSTable which promotes itself to a private writable file on demand+ */+ public static SSTableHandleMap openWorkingCopy(final int keyLength,+ final ByteOrder ordering, final int valueLength,+ final File sourceFingerprint) throws IOException {+ if (sourceFingerprint == null) throw new NullPointerException("sourceFingerprint");+ if (!sourceFingerprint.isFile()) {+ throw new IOException("Fingerprint index does not exist: " + sourceFingerprint);+ }+ final File directoryFile = sourceFingerprint.getAbsoluteFile().getParentFile();+ if (directoryFile == null) {+ throw new IOException("Fingerprint index has no parent directory: " + sourceFingerprint);+ }+ final Path directory = directoryFile.toPath();+ final String workingPrefix = workingPrefix(sourceFingerprint);++ if (!sourceFingerprint.getName().endsWith(".gz")) {+ return new SSTableHandleMap(+ keyLength, ordering, valueLength, sourceFingerprint,+ false, null, null, false, directory, workingPrefix);+ }++ Path workingFile = null;+ try {+ workingFile = Files.createTempFile(directory, workingPrefix, ".sstwork");+ copyFingerprint(sourceFingerprint.toPath(), workingFile);+ final SSTableHandleMap result = openOwnedWorkingFile(+ keyLength, ordering, valueLength, workingFile.toFile());++ /* Ownership has moved to the returned map. */+ workingFile = null;+ return result;+ } finally {+ if (workingFile != null) Files.deleteIfExists(workingFile);+ }+ }++ /**+ * Map a completed private table before taking its ownership lock. On POSIX,+ * closing any descriptor for a file can release process-scoped record locks;+ * therefore no copy or mapping channel may be opened after this lock is taken.+ */+ private static SSTableHandleMap openOwnedWorkingFile(+ final int keyLength, final ByteOrder ordering,+ final int valueLength, final File workingFile) throws IOException {+ final SSTableHandleMap result = new SSTableHandleMap(+ keyLength, ordering, valueLength, workingFile, true,+ null, null, true, null, null);+ FileChannel ownershipChannel = null;+ FileLock ownershipLock = null;+ boolean complete = false;+ try {+ ownershipChannel = FileChannel.open(workingFile.toPath(),+ StandardOpenOption.READ, StandardOpenOption.WRITE);+ ownershipLock = ownershipChannel.tryLock();+ if (ownershipLock == null) {+ throw new IOException("Cannot lock SSTable working file " + workingFile);+ }+ result.ownershipChannel = ownershipChannel;+ result.ownershipLock = ownershipLock;+ complete = true;+ return result;+ } finally {+ if (!complete) {+ if (ownershipLock != null) {+ try {+ ownershipLock.release();+ } catch (final IOException ignored) {+ }+ }+ if (ownershipChannel != null) {+ try {+ ownershipChannel.close();+ } catch (final IOException ignored) {+ }+ }+ result.table.close();+ }+ }+ }++ private static void copyFingerprint(final Path source, final Path target) throws IOException {+ try (InputStream input = openFingerprintInput(source);+ OutputStream output = new BufferedOutputStream(Files.newOutputStream(target,+ StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING),+ COPY_BUFFER_SIZE)) {+ final byte[] buffer = new byte[COPY_BUFFER_SIZE];+ int read;+ while ((read = input.read(buffer)) >= 0) {+ if (read > 0) output.write(buffer, 0, read);+ }+ }+ }++ private static InputStream openFingerprintInput(final Path source) throws IOException {+ final BufferedInputStream input = new BufferedInputStream(+ Files.newInputStream(source), COPY_BUFFER_SIZE);+ if (!source.getFileName().toString().endsWith(".gz")) return input;+ try {+ return new GZIPInputStream(input, COPY_BUFFER_SIZE);+ } catch (final IOException e) {+ try {+ input.close();+ } catch (final IOException closeFailure) {+ e.addSuppressed(closeFailure);+ }+ throw e;+ }+ }++ private static String workingPrefix(final File fingerprint) {+ return "sst-work-" + Integer.toHexString(+ fingerprint.getAbsolutePath().hashCode()) + "-";+ }++ /**+ * Remove private SSTable generations and builder files left by a crashed process.+ *+ * <p>The supplied roots are normalized and overlapping roots are scanned only+ * once. The walk does not follow symbolic links, so recovery cannot escape a+ * configured storage tree. Live tables lock their working file, while an active+ * builder locks one marker shared by all files of its build session. Such files+ * are counted as in use and are never removed. Legacy builder artifacts from+ * versions without a session marker are also recognized; recovery is therefore+ * intended to run before repositories start building indexes. All other I/O+ * failures are recorded instead of aborting application startup.</p>+ *+ * <p>This operation is intended to run once before repositories are opened. A+ * normal close still removes its own file immediately.</p>+ *+ * @param roots repository roots which may contain temporary SSTable artifacts+ * @return counts describing the best-effort recovery pass+ */+ public static RecoveryReport recoverWorkingCopies(final File... roots) {+ if (roots == null) throw new NullPointerException("roots");+ final RecoveryReport report = new RecoveryReport();+ final ArrayList<Path> scanRoots = new ArrayList<>();+ for (final File root : roots) {+ if (root == null) continue;+ final Path configured = root.toPath().toAbsolutePath().normalize();+ final Path candidate;+ try {+ /* Resolve an explicitly configured symlink root, but walkFileTree()+ * still does not follow any symlink found below that root. */+ candidate = configured.toRealPath();+ } catch (final NoSuchFileException e) {+ /* A not-yet-created configured directory simply has nothing to recover. */+ continue;+ } catch (final IOException | SecurityException e) {+ report.failures++;+ continue;+ }+ if (!Files.isDirectory(candidate)) continue;+ boolean covered = false;+ for (final Path existing : new ArrayList<>(scanRoots)) {+ if (candidate.startsWith(existing)) {+ covered = true;+ break;+ }+ if (existing.startsWith(candidate)) scanRoots.remove(existing);+ }+ if (!covered) scanRoots.add(candidate);+ }++ for (final Path root : scanRoots) {+ try {+ Files.walkFileTree(root, new SimpleFileVisitor<Path>() {+ @Override+ public FileVisitResult visitFile(final Path file,+ final BasicFileAttributes attributes) {+ if (attributes.isRegularFile()+ && isRecoverableArtifact(file.getFileName().toString())) {+ report.candidates++;+ recoverArtifact(file, report);+ }+ return FileVisitResult.CONTINUE;+ }++ @Override+ public FileVisitResult visitFileFailed(final Path file,+ final IOException failure) {+ report.failures++;+ return FileVisitResult.CONTINUE;+ }+ });+ } catch (final IOException | SecurityException e) {+ report.failures++;+ }+ }+ return report;+ }++ private static boolean isRecoverableArtifact(final String name) {+ return WORKING_ARTIFACT.matcher(name).matches()+ || BUILD_ARTIFACT.matcher(name).matches()+ || BUILD_LOCK_ARTIFACT.matcher(name).matches()+ || LEGACY_BUILD_ARTIFACT.matcher(name).matches();+ }++ private static void recoverArtifact(+ final Path file, final RecoveryReport report) {+ final Matcher buildArtifact = BUILD_ARTIFACT.matcher(+ file.getFileName().toString());+ final Path sessionLock = buildArtifact.matches()+ ? file.resolveSibling(buildArtifact.group(1) + BUILD_LOCK_SUFFIX)+ : null;+ /* A finished Builder result outlives its session marker and owns a lock on+ * the table itself. Intermediate files use the shared session marker. */+ final Path lockFile = sessionLock != null+ && Files.isRegularFile(sessionLock, LinkOption.NOFOLLOW_LINKS)+ ? sessionLock : file;++ try (FileChannel channel = FileChannel.open(lockFile,+ StandardOpenOption.READ, StandardOpenOption.WRITE)) {+ FileLock lock = null;+ try {+ lock = channel.tryLock();+ if (lock == null) {+ report.inUse++;+ return;+ }+ } catch (final OverlappingFileLockException e) {+ report.inUse++;+ return;+ } finally {+ if (lock != null) lock.release();+ }+ } catch (final IOException | SecurityException e) {+ report.failures++;+ return;+ }++ try {+ if (Files.deleteIfExists(file)) report.deleted++;+ } catch (final IOException | SecurityException e) {+ report.failures++;+ }+ }++ /**+ * Detach this table from its published fingerprint before the heap generation+ * is changed or the fingerprint is deleted.+ *+ * <p>This method does not change a key or offset. It only establishes a private+ * writable backing generation. HeapModifier calls it before invalidating the+ * fingerprint, while the mutating map methods also call it defensively so direct+ * users cannot accidentally write into a read-only mapping.</p>+ *+ * @throws IOException when the private generation cannot be created completely+ */+ public void prepareForMutation() throws IOException {+ this.backingLock.writeLock().lock();+ try {+ requireOpen();+ promoteToWorkingCopy();+ } finally {+ this.backingLock.writeLock().unlock();+ }+ }++ private void promoteToWorkingCopy() throws IOException {+ if (this.writable) return;+ if (this.promotionDirectory == null || this.promotionPrefix == null) {+ throw new IOException("Read-only SSTable has no working-file configuration: "+ + this.tableFile);+ }++ Path workingFile = null;+ FileChannel workingChannel = null;+ FileLock workingLock = null;+ ChunkedBytes workingTable = null;+ try {+ workingFile = Files.createTempFile(+ this.promotionDirectory, this.promotionPrefix, ".sstwork");++ /*+ * Copy through the mapping, not through the fingerprint path. The heap+ * may already have invalidated that path in an exceptional recovery flow;+ * the mapped generation remains the authoritative index until this switch.+ */+ copyTable(this.table, workingFile);+ workingTable = mapFile(workingFile.toFile(), true);+ /* Take ownership only after every other descriptor used for copying+ * and mapping has been closed; see openOwnedWorkingFile(). */+ workingChannel = FileChannel.open(workingFile,+ StandardOpenOption.READ, StandardOpenOption.WRITE);+ workingLock = workingChannel.tryLock();+ if (workingLock == null) {+ throw new IOException("Cannot lock SSTable working file " + workingFile);+ }++ final ChunkedBytes publishedTable = this.table;+ this.table = workingTable;+ this.tableFile = workingFile.toFile().getAbsoluteFile();+ this.writable = true;+ this.deleteOnClose = true;+ this.ownershipChannel = workingChannel;+ this.ownershipLock = workingLock;++ /* Ownership moved to this instance before the published mapping closes. */+ workingTable = null;+ workingFile = null;+ workingChannel = null;+ workingLock = null;+ publishedTable.close();+ } finally {+ if (workingTable != null) workingTable.close();+ if (workingLock != null) {+ try {+ workingLock.release();+ } catch (final IOException ignored) {+ }+ }+ if (workingChannel != null) {+ try {+ workingChannel.close();+ } catch (final IOException ignored) {+ }+ }+ if (workingFile != null) Files.deleteIfExists(workingFile);+ }+ }++ private static void copyTable(final ChunkedBytes source, final Path target)+ throws IOException {+ try (InputStream input = source.openStream();+ OutputStream output = new BufferedOutputStream(Files.newOutputStream(target,+ StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING),+ COPY_BUFFER_SIZE)) {+ final byte[] buffer = new byte[COPY_BUFFER_SIZE];+ int read;+ while ((read = input.read(buffer)) >= 0) {+ if (read > 0) output.write(buffer, 0, read);+ }+ }+ }++ /**+ * Builds a sorted table from an unsorted stream while keeping Java heap use bounded.+ * Records are sorted in fixed-size memory runs and then merged with a bounded fan-in.+ * The returned table owns its temporary backing file and removes it on close.+ */+ public static final class Builder implements AutoCloseable {++ private final int keyLength;+ private final int valueLength;+ private final int recordLength;+ private final ByteOrder ordering;+ private final Path directory;+ private final String temporaryPrefix;+ private final int maxBufferedRecords;+ private final ArrayList<byte[]> buffer;+ private final ArrayList<Path> runs;+ private final ArrayList<Path> ownedFiles;+ private final Path sessionLockFile;+ private final FileChannel sessionLockChannel;+ private final FileLock sessionLock;++ private boolean finished;+ private boolean closed;++ public Builder(final int keyLength, final ByteOrder ordering,+ final int valueLength, final File directory, final String name) throws IOException {+ this(keyLength, ordering, valueLength, directory, name, DEFAULT_RUN_MEMORY_BYTES);+ }++ /**+ * @param maxRunMemoryBytes approximate upper bound for record payload and array overhead+ */+ public Builder(final int keyLength, final ByteOrder ordering,+ final int valueLength, final File directory, final String name,+ final int maxRunMemoryBytes) throws IOException {+ if (keyLength <= 0) throw new IllegalArgumentException("keyLength must be positive");+ if (valueLength != Long.BYTES) {+ throw new IllegalArgumentException("SSTable offsets require valueLength 8");+ }+ if (ordering == null) throw new NullPointerException("ordering");+ if (directory == null) throw new NullPointerException("directory");+ if (maxRunMemoryBytes <= 0) {+ throw new IllegalArgumentException("maxRunMemoryBytes must be positive");+ }+ if (!directory.exists() && !directory.mkdirs()) {+ throw new IOException("Cannot create SSTable build directory: " + directory);+ }+ if (!directory.isDirectory()) {+ throw new IOException("SSTable build path is not a directory: " + directory);+ }++ this.keyLength = keyLength;+ this.valueLength = valueLength;+ this.recordLength = Math.addExact(keyLength, valueLength);+ this.ordering = ordering;+ this.directory = directory.toPath();+ final long estimatedRecordBytes = this.recordLength + ESTIMATED_ARRAY_OVERHEAD;+ this.maxBufferedRecords = (int) Math.max(1L, Math.min(Integer.MAX_VALUE,+ maxRunMemoryBytes / estimatedRecordBytes));+ this.buffer = new ArrayList<>(this.maxBufferedRecords);+ this.runs = new ArrayList<>();+ this.ownedFiles = new ArrayList<>();+ this.finished = false;+ this.closed = false;++ final String sessionName = "sst-build-" + Integer.toHexString(+ name == null ? 0 : name.hashCode()) + "-"+ + UUID.randomUUID().toString().replace("-", "");+ this.temporaryPrefix = sessionName + "-";+ this.sessionLockFile = this.directory.resolve(sessionName + BUILD_LOCK_SUFFIX);+ FileChannel lockChannel = null;+ FileLock lock = null;+ try {+ lockChannel = FileChannel.open(this.sessionLockFile,+ StandardOpenOption.CREATE_NEW,+ StandardOpenOption.READ, StandardOpenOption.WRITE);+ lock = lockChannel.tryLock();+ if (lock == null) {+ throw new IOException("Cannot lock SSTable build session "+ + this.sessionLockFile);+ }+ } catch (final IOException | RuntimeException failure) {+ if (lock != null) {+ try {+ lock.release();+ } catch (final IOException closeFailure) {+ failure.addSuppressed(closeFailure);+ }+ }+ if (lockChannel != null) {+ try {+ lockChannel.close();+ } catch (final IOException closeFailure) {+ failure.addSuppressed(closeFailure);+ }+ }+ try {+ Files.deleteIfExists(this.sessionLockFile);+ } catch (final IOException cleanupFailure) {+ failure.addSuppressed(cleanupFailure);+ }+ throw failure;+ }+ this.sessionLockChannel = lockChannel;+ this.sessionLock = lock;+ }++ public void consume(final byte[] key, final long offset) throws IOException {+ requireBuilding();+ if (key == null) throw new NullPointerException("key");+ if (offset < 0) throw new IllegalArgumentException("offset must not be negative");++ final byte[] record = new byte[this.recordLength];+ System.arraycopy(key, 0, record, 0, Math.min(key.length, this.keyLength));+ NaturalOrder.encodeLong(offset, record, this.keyLength, this.valueLength);+ this.buffer.add(record);+ if (this.buffer.size() >= this.maxBufferedRecords) flushRun();+ }++ public SSTableHandleMap finish() throws IOException {+ requireBuilding();+ flushRun();++ List<Path> current = new ArrayList<>(this.runs);+ this.runs.clear();+ while (current.size() > MAX_MERGE_FAN_IN) {+ final ArrayList<Path> next = new ArrayList<>(+ (current.size() + MAX_MERGE_FAN_IN - 1) / MAX_MERGE_FAN_IN);+ for (int start = 0; start < current.size(); start += MAX_MERGE_FAN_IN) {+ final int end = Math.min(current.size(), start + MAX_MERGE_FAN_IN);+ if (end - start == 1) {+ next.add(current.get(start));+ continue;+ }+ final List<Path> group = new ArrayList<>(current.subList(start, end));+ final Path merged = createTemporaryFile(".merge");+ mergeRuns(group, merged);+ for (final Path source : group) deleteOwned(source);+ next.add(merged);+ }+ current = next;+ }++ final Path resultFile;+ if (current.isEmpty()) {+ resultFile = createTemporaryFile(".sst");+ } else if (current.size() == 1) {+ resultFile = current.get(0);+ } else {+ resultFile = createTemporaryFile(".sst");+ mergeRuns(current, resultFile);+ for (final Path source : current) deleteOwned(source);+ }++ final SSTableHandleMap result = openOwnedWorkingFile(+ this.keyLength, this.ordering, this.valueLength,+ resultFile.toFile());+ this.ownedFiles.remove(resultFile);+ this.finished = true;+ return result;+ }++ private void flushRun() throws IOException {+ if (this.buffer.isEmpty()) return;+ this.buffer.sort((left, right) -> this.ordering.compare(+ left, 0, right, 0, this.keyLength));+ final Path run = createTemporaryFile(".run");+ boolean complete = false;+ try (OutputStream output = new BufferedOutputStream(+ new FileOutputStream(run.toFile()), COPY_BUFFER_SIZE)) {+ byte[] pending = null;+ for (final byte[] record : this.buffer) {+ if (pending == null) {+ pending = record;+ } else if (sameKey(pending, record)) {+ if (readOffset(record) > readOffset(pending)) pending = record;+ } else {+ output.write(pending);+ pending = record;+ }+ }+ if (pending != null) output.write(pending);+ complete = true;+ } finally {+ this.buffer.clear();+ if (!complete) deleteOwned(run);+ }+ this.runs.add(run);+ }++ private void mergeRuns(final List<Path> sources, final Path target) throws IOException {+ final ArrayList<RunCursor> cursors = new ArrayList<>(sources.size());+ IOException failure = null;+ try (OutputStream output = new BufferedOutputStream(+ new FileOutputStream(target.toFile()), COPY_BUFFER_SIZE)) {+ final Comparator<RunCursor> comparator = (left, right) -> {+ final int compared = this.ordering.compare(left.record, 0,+ right.record, 0, this.keyLength);+ return compared == 0 ? Integer.compare(left.ordinal, right.ordinal) : compared;+ };+ final PriorityQueue<RunCursor> queue = new PriorityQueue<>(comparator);+ int ordinal = 0;+ for (final Path source : sources) {+ final RunCursor cursor = new RunCursor(source, ordinal++);+ cursors.add(cursor);+ if (cursor.advance()) queue.add(cursor);+ }++ final byte[] mergedRecord = new byte[this.recordLength];+ while (!queue.isEmpty()) {+ RunCursor cursor = queue.poll();+ System.arraycopy(cursor.record, 0, mergedRecord, 0, this.keyLength);+ long newestOffset = readOffset(cursor.record);+ if (cursor.advance()) queue.add(cursor);++ while (!queue.isEmpty() && sameKey(mergedRecord, queue.peek().record)) {+ cursor = queue.poll();+ newestOffset = Math.max(newestOffset, readOffset(cursor.record));+ if (cursor.advance()) queue.add(cursor);+ }+ NaturalOrder.encodeLong(newestOffset, mergedRecord,+ this.keyLength, this.valueLength);+ output.write(mergedRecord);+ }+ } catch (final IOException e) {+ failure = e;+ throw e;+ } finally {+ IOException closeFailure = null;+ for (final RunCursor cursor : cursors) {+ try {+ cursor.close();+ } catch (final IOException e) {+ if (failure != null) failure.addSuppressed(e);+ else if (closeFailure == null) closeFailure = e;+ else closeFailure.addSuppressed(e);+ }+ }+ if (failure == null && closeFailure != null) throw closeFailure;+ }+ }++ private boolean sameKey(final byte[] left, final byte[] right) {+ return this.ordering.equal(left, 0, right, 0, this.keyLength);+ }++ private long readOffset(final byte[] record) {+ return NaturalOrder.decodeLong(record, this.keyLength, this.valueLength);+ }++ private Path createTemporaryFile(final String suffix) throws IOException {+ final Path file = Files.createTempFile(+ this.directory, this.temporaryPrefix, suffix);+ this.ownedFiles.add(file);+ return file;+ }++ private void deleteOwned(final Path file) throws IOException {+ Files.deleteIfExists(file);+ this.ownedFiles.remove(file);+ }++ private void requireBuilding() {+ if (this.closed) throw new IllegalStateException("SSTable builder is closed");+ if (this.finished) throw new IllegalStateException("SSTable builder is already finished");+ }++ @Override+ public void close() throws IOException {+ if (this.closed) return;+ this.closed = true;+ this.buffer.clear();+ IOException failure = null;+ for (final Path file : new ArrayList<>(this.ownedFiles)) {+ try {+ deleteOwned(file);+ } catch (final IOException e) {+ if (failure == null) failure = e;+ else failure.addSuppressed(e);+ }+ }+ if (this.sessionLock.isValid()) {+ try {+ this.sessionLock.release();+ } catch (final IOException e) {+ if (failure == null) failure = e;+ else failure.addSuppressed(e);+ }+ }+ try {+ this.sessionLockChannel.close();+ } catch (final IOException e) {+ if (failure == null) failure = e;+ else failure.addSuppressed(e);+ }+ try {+ Files.deleteIfExists(this.sessionLockFile);+ } catch (final IOException e) {+ if (failure == null) failure = e;+ else failure.addSuppressed(e);+ }+ if (failure != null) throw failure;+ }++ private final class RunCursor implements AutoCloseable {+ private final InputStream input;+ private final int ordinal;+ private final byte[] record;++ private RunCursor(final Path run, final int ordinal) throws IOException {+ this.input = new BufferedInputStream(+ Files.newInputStream(run), COPY_BUFFER_SIZE);+ this.ordinal = ordinal;+ this.record = new byte[Builder.this.recordLength];+ }++ private boolean advance() throws IOException {+ final int first = this.input.read();+ if (first < 0) return false;+ this.record[0] = (byte) first;+ int offset = 1;+ while (offset < this.record.length) {+ final int read = this.input.read(+ this.record, offset, this.record.length - offset);+ if (read < 0) {+ throw new EOFException("Truncated SSTable run record");+ }+ if (read == 0) continue;+ offset += read;+ }+ return true;+ }++ @Override+ public void close() throws IOException {+ this.input.close();+ }+ }+ }++ @Override+ public long mem() {+ /* File-backed pages belong to the OS page cache, not the Java heap. */+ return 0L;+ }++ @Override+ public void optimize() {+ /* Immutable sorted storage has no heap capacity to trim. */+ }++ @Override+ public int dump(final File file) throws IOException {+ requireOpen();+ if (file == null) throw new NullPointerException("file");+ if (this.tableFile.equals(file.getAbsoluteFile())) {+ if (this.liveCount.get() == this.recordCount) return this.recordCount;+ throw new IOException("Cannot compact an open SSTable in place: " + file);+ }++ final File parent = file.getAbsoluteFile().getParentFile();+ if (parent != null && !parent.exists() && !parent.mkdirs()) {+ throw new IOException("Cannot create SSTable dump directory: " + parent);+ }+ final File temporary = new File(parent, file.getName() + ".prt");+ int written = 0;+ final byte[] record = new byte[this.recordLength];+ try (FileOutputStream fileStream = new FileOutputStream(temporary);+ OutputStream buffered = new BufferedOutputStream(fileStream, COPY_BUFFER_SIZE);+ OutputStream output = file.getName().endsWith(".gz")+ ? new GZIPOutputStream(buffered, COPY_BUFFER_SIZE) {{+ this.def.setLevel(Deflater.BEST_COMPRESSION);+ }}+ : buffered) {+ for (int ordinal = 0; ordinal < this.recordCount; ordinal++) {+ if (isDeleted(ordinal)) continue;+ readRecord(ordinal, record);+ output.write(record);+ written++;+ }+ } catch (final IOException | RuntimeException e) {+ Files.deleteIfExists(temporary.toPath());+ if (e instanceof IOException) throw (IOException) e;+ throw e;+ }++ try {+ Files.move(temporary.toPath(), file.toPath(), StandardCopyOption.ATOMIC_MOVE,+ StandardCopyOption.REPLACE_EXISTING);+ } catch (final AtomicMoveNotSupportedException e) {+ try {+ Files.move(temporary.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING);+ } catch (final IOException moveFailure) {+ try {+ Files.deleteIfExists(temporary.toPath());+ } catch (final IOException cleanupFailure) {+ moveFailure.addSuppressed(cleanupFailure);+ }+ throw moveFailure;+ }+ } catch (final IOException moveFailure) {+ try {+ Files.deleteIfExists(temporary.toPath());+ } catch (final IOException cleanupFailure) {+ moveFailure.addSuppressed(cleanupFailure);+ }+ throw moveFailure;+ }+ return written;+ }++ @Override+ public synchronized void clear() {+ this.backingLock.writeLock().lock();+ try {+ requireOpen();+ if (this.liveCount.get() == 0) return;+ promoteToWorkingCopy();+ for (int ordinal = 0; ordinal < this.recordCount; ordinal++) {+ if (!isDeleted(ordinal)) writeDeletedOffset(ordinal);+ }+ this.liveCount.set(0);+ } catch (final IOException e) {+ throw new UncheckedIOException(e);+ } finally {+ this.backingLock.writeLock().unlock();+ }+ }++ @Override+ public byte[] smallestKey() {+ requireOpen();+ for (int ordinal = 0; ordinal < this.recordCount; ordinal++) {+ if (!isDeleted(ordinal)) return readKey(ordinal);+ }+ return null;+ }++ @Override+ public byte[] largestKey() {+ requireOpen();+ for (int ordinal = this.recordCount - 1; ordinal >= 0; ordinal--) {+ if (!isDeleted(ordinal)) return readKey(ordinal);+ }+ return null;+ }++ @Override+ public boolean has(final byte[] key) {+ return get(key) >= 0;+ }++ @Override+ public long get(final byte[] key) {+ requireOpen();+ final int ordinal = find(key);+ return ordinal < 0 || isDeleted(ordinal) ? -1 : readValue(ordinal);+ }++ @Override+ public ArrayList<long[]> removeDoubles() throws SpaceExceededException {+ /* An SSTable is required to contain unique sorted keys. */+ return new ArrayList<>();+ }++ @Override+ public ArrayList<byte[]> top(final int count) {+ requireOpen();+ final ArrayList<byte[]> result = new ArrayList<>(Math.max(0, Math.min(count, size())));+ if (count <= 0) return result;+ for (int ordinal = this.recordCount - 1;+ ordinal >= 0 && result.size() < count; ordinal--) {+ if (!isDeleted(ordinal)) result.add(readKey(ordinal));+ }+ return result;+ }++ @Override+ public synchronized long remove(final byte[] key) {+ this.backingLock.writeLock().lock();+ try {+ requireOpen();+ final int ordinal = find(key);+ if (ordinal < 0 || isDeleted(ordinal)) return -1;+ final long previous = readValue(ordinal);+ promoteToWorkingCopy();+ writeDeletedOffset(ordinal);+ this.liveCount.decrementAndGet();+ return previous;+ } catch (final IOException e) {+ throw new UncheckedIOException(e);+ } finally {+ this.backingLock.writeLock().unlock();+ }+ }++ @Override+ public synchronized long removeone() {+ this.backingLock.writeLock().lock();+ try {+ requireOpen();+ for (int ordinal = this.recordCount - 1; ordinal >= 0; ordinal--) {+ if (isDeleted(ordinal)) continue;+ final long previous = readValue(ordinal);+ promoteToWorkingCopy();+ writeDeletedOffset(ordinal);+ this.liveCount.decrementAndGet();+ return previous;+ }+ return -1;+ } catch (final IOException e) {+ throw new UncheckedIOException(e);+ } finally {+ this.backingLock.writeLock().unlock();+ }+ }++ @Override+ public int size() {+ requireOpen();+ return this.liveCount.get();+ }++ @Override+ public boolean isEmpty() {+ return size() == 0;+ }++ @Override+ public CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) {+ requireOpen();+ return new KeyIterator(up, firstKey);+ }++ @Override+ public Iterator<Map.Entry<byte[], Long>> iterator() {+ requireOpen();+ return new EntryIterator();+ }++ @Override+ public synchronized void close() {+ this.backingLock.writeLock().lock();+ try {+ if (this.closed) return;+ this.closed = true;+ this.table.close();++ IOException failure = null;+ if (this.writable && this.ownershipChannel != null) {+ try {+ this.ownershipChannel.force(true);+ } catch (final IOException e) {+ failure = e;+ }+ } else if (this.writable) {+ try (FileChannel channel = FileChannel.open(+ this.tableFile.toPath(), StandardOpenOption.WRITE)) {+ channel.force(true);+ } catch (final IOException e) {+ failure = e;+ }+ }++ /* Release ownership only after all mapped writes have been flushed. */+ if (this.ownershipLock != null && this.ownershipLock.isValid()) {+ try {+ this.ownershipLock.release();+ } catch (final IOException e) {+ if (failure == null) failure = e;+ else failure.addSuppressed(e);+ }+ }+ if (this.ownershipChannel != null) {+ try {+ this.ownershipChannel.close();+ } catch (final IOException e) {+ if (failure == null) failure = e;+ else failure.addSuppressed(e);+ }+ }++ if (this.deleteOnClose) {+ try {+ Files.deleteIfExists(this.tableFile.toPath());+ } catch (final IOException e) {+ this.tableFile.deleteOnExit();+ if (failure == null) failure = e;+ else failure.addSuppressed(e);+ }+ }+ if (failure != null) {+ throw new UncheckedIOException(+ "Cannot close SSTable " + this.tableFile, failure);+ }+ } finally {+ this.backingLock.writeLock().unlock();+ }+ }++ private int find(final byte[] sourceKey) {+ final byte[] key = normalizeKey(sourceKey);+ final byte[] pivotKey = new byte[this.keyLength];+ int low = 0;+ int high = this.recordCount;+ while (low < high) {+ final int pivot = (low + high) >>> 1;+ readKey(pivot, pivotKey);+ final int comparison = this.ordering.compare(pivotKey, 0, key, 0, this.keyLength);+ if (comparison < 0) low = pivot + 1;+ else high = pivot;+ }+ if (low >= this.recordCount) return -1;+ readKey(low, pivotKey);+ return this.ordering.equal(pivotKey, 0, key, 0, this.keyLength) ? low : -1;+ }++ private int lowerBound(final byte[] sourceKey) {+ if (sourceKey == null || sourceKey.length == 0) return 0;+ final byte[] key = normalizeKey(sourceKey);+ final byte[] pivotKey = new byte[this.keyLength];+ int low = 0;+ int high = this.recordCount;+ while (low < high) {+ final int pivot = (low + high) >>> 1;+ readKey(pivot, pivotKey);+ if (this.ordering.compare(pivotKey, 0, key, 0, this.keyLength) < 0) low = pivot + 1;+ else high = pivot;+ }+ return low;+ }++ private byte[] normalizeKey(final byte[] key) {+ if (key == null) throw new NullPointerException("key");+ if (key.length == this.keyLength) return key;+ final byte[] normalized = new byte[this.keyLength];+ System.arraycopy(key, 0, normalized, 0, Math.min(key.length, this.keyLength));+ return normalized;+ }++ private byte[] readKey(final int ordinal) {+ final byte[] key = new byte[this.keyLength];+ readKey(ordinal, key);+ return key;+ }++ private void readKey(final int ordinal, final byte[] key) {+ readTable(recordOffset(ordinal), key, 0, this.keyLength);+ }++ private long readValue(final int ordinal) {+ final byte[] value = new byte[this.valueLength];+ readTable(recordOffset(ordinal) + this.keyLength,+ value, 0, this.valueLength);+ return NaturalOrder.decodeLong(value, 0, value.length);+ }++ private void readRecord(final int ordinal, final byte[] record) {+ readTable(recordOffset(ordinal), record, 0, this.recordLength);+ }++ private long recordOffset(final int ordinal) {+ if (ordinal < 0 || ordinal >= this.recordCount) {+ throw new IndexOutOfBoundsException("record ordinal " + ordinal);+ }+ return (long) ordinal * this.recordLength;+ }++ private boolean isDeleted(final int ordinal) {+ /* With an eight-byte big-endian value, the first bit is the sign bit. */+ this.backingLock.readLock().lock();+ try {+ return this.table.get(recordOffset(ordinal) + this.keyLength) < 0;+ } finally {+ this.backingLock.readLock().unlock();+ }+ }++ private void writeDeletedOffset(final int ordinal) throws IOException {+ final long offset = recordOffset(ordinal) + this.keyLength;+ if (this.table.write(offset, DELETED_OFFSET, 0, DELETED_OFFSET.length)+ != DELETED_OFFSET.length) {+ throw new IOException("Cannot persist deleted SSTable offset at record " + ordinal);+ }+ }++ private void requireOpen() {+ if (this.closed) throw new IllegalStateException("SSTableHandleMap is closed: " + this.tableFile);+ }++ private void readTable(final long position, final byte[] target,+ final int offset, final int length) {+ this.backingLock.readLock().lock();+ try {+ readFully(this.table, position, target, offset, length);+ } finally {+ this.backingLock.readLock().unlock();+ }+ }++ private static ChunkedBytes mapFile(final File file, final boolean writable) throws IOException {+ final ChunkedBytes bytes = new ChunkedBytes();+ try {+ bytes.appendFile(file.toPath(), writable);+ return bytes;+ } catch (final RuntimeException e) {+ bytes.close();+ if (e.getCause() instanceof IOException) throw (IOException) e.getCause();+ throw new IOException("Cannot map SSTable file " + file, e);+ }+ }++ private static void readFully(final ChunkedBytes bytes, final long position,+ final byte[] target, final int offset, final int length) {+ final int read = bytes.read(position, target, offset, length);+ if (read != length) {+ throw new IllegalStateException("Short SSTable read at " + position+ + ": expected " + length + ", got " + read);+ }+ }++ private final class KeyIterator implements CloneableIterator<byte[]> {++ private final boolean up;+ private int nextOrdinal;+ private int preparedOrdinal;+ private byte[] lastKey;++ private KeyIterator(final boolean up, final byte[] firstKey) {+ this.up = up;+ this.nextOrdinal = up ? lowerBound(firstKey) : SSTableHandleMap.this.recordCount - 1;+ this.preparedOrdinal = -1;+ this.lastKey = null;+ }++ @Override+ public boolean hasNext() {+ requireOpen();+ if (this.preparedOrdinal >= 0) return true;+ int ordinal = this.nextOrdinal;+ while (ordinal >= 0 && ordinal < SSTableHandleMap.this.recordCount+ && isDeleted(ordinal)) ordinal += this.up ? 1 : -1;+ if (ordinal < 0 || ordinal >= SSTableHandleMap.this.recordCount) return false;+ this.preparedOrdinal = ordinal;+ return true;+ }++ @Override+ public byte[] next() {+ if (!hasNext()) throw new NoSuchElementException();+ final int ordinal = this.preparedOrdinal;+ this.preparedOrdinal = -1;+ this.nextOrdinal = ordinal + (this.up ? 1 : -1);+ this.lastKey = readKey(ordinal);+ return this.lastKey;+ }++ @Override+ public void remove() {+ if (this.lastKey == null) throw new IllegalStateException();+ SSTableHandleMap.this.remove(this.lastKey);+ this.lastKey = null;+ }++ @Override+ public CloneableIterator<byte[]> clone(final Object modifier) {+ return new KeyIterator(this.up, (byte[]) modifier);+ }++ @Override+ public void close() {+ }+ }++ private final class EntryIterator implements Iterator<Map.Entry<byte[], Long>> {++ private int nextOrdinal = 0;+ private int preparedOrdinal = -1;+ private byte[] lastKey;++ @Override+ public boolean hasNext() {+ requireOpen();+ if (this.preparedOrdinal >= 0) return true;+ int ordinal = this.nextOrdinal;+ while (ordinal < SSTableHandleMap.this.recordCount && isDeleted(ordinal)) ordinal++;+ if (ordinal >= SSTableHandleMap.this.recordCount) return false;+ this.preparedOrdinal = ordinal;+ return true;+ }++ @Override+ public Map.Entry<byte[], Long> next() {+ if (!hasNext()) throw new NoSuchElementException();+ final int ordinal = this.preparedOrdinal;+ this.preparedOrdinal = -1;+ this.nextOrdinal = ordinal + 1;+ this.lastKey = readKey(ordinal);+ return new AbstractMap.SimpleImmutableEntry<>(this.lastKey, readValue(ordinal));+ }++ @Override+ public void remove() {+ if (this.lastKey == null) throw new IllegalStateException();+ SSTableHandleMap.this.remove(this.lastKey);+ this.lastKey = null;+ }+ }+}diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.javaindex ce735ca75..9e10f8032 100644--- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java+++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java@@ -35,7 +35,7 @@ import net.yacy.cora.sorting.Rating;import net.yacy.cora.util.ConcurrentLog;import net.yacy.cora.util.SpaceExceededException;import net.yacy.kelondro.blob.ArrayStack;-import net.yacy.kelondro.blob.BLOB;+import net.yacy.kelondro.blob.ImmutableBLOB;import net.yacy.kelondro.data.word.Word;import net.yacy.kelondro.index.Row;import net.yacy.kelondro.index.RowSet;@@ -364,7 +364,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {return this.array.reduce(termHash, new BLOBReducer(termHash, reducer));}- public class BLOBReducer implements BLOB.Reducer {+ public class BLOBReducer implements ImmutableBLOB.Reducer {ContainerReducer<ReferenceType> rewriter;byte[] wordHash;diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.javaindex 8b4e731f8..f88ba737a 100644--- a/source/net/yacy/search/Switchboard.java+++ b/source/net/yacy/search/Switchboard.java@@ -185,6 +185,7 @@ import net.yacy.kelondro.blob.Tables;import net.yacy.kelondro.blob.Tables.SortDirection;import net.yacy.kelondro.data.meta.URIMetadataNode;import net.yacy.kelondro.data.word.Word;+import net.yacy.kelondro.index.SSTableHandleMap;import net.yacy.kelondro.logging.GuiHandler;import net.yacy.kelondro.logging.ThreadDump;import net.yacy.kelondro.rwi.ReferenceContainer;@@ -366,6 +367,25 @@ public final class Switchboard extends serverSwitch {this.log.config("Index Primary Path: " + indexPath.toString());final File archivePath = this.getDataPath(SwitchboardConstants.INDEX_ARCHIVE_PATH, SwitchboardConstants.INDEX_ARCHIVE_DEFAULT);this.log.config("Index Archive Path: " + archivePath.toString());+ this.htCachePath =+ this.getDataPath(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT);++ /*+ * SSTable working and builder files are private scratch state, never a+ * published fingerprint. A hard process stop can prevent normal cleanup.+ * Recover all configured repository trees before any ArrayStack opens them;+ * exclusive file locks protect tables still owned by another YaCy process.+ */+ final SSTableHandleMap.RecoveryReport sstableRecovery =+ SSTableHandleMap.recoverWorkingCopies(+ dataPath, indexPath, archivePath, this.htCachePath);+ if (sstableRecovery.candidates() > 0 || sstableRecovery.failures() > 0) {+ this.log.info("SSTable crash recovery: deleted "+ + sstableRecovery.deleted() + " of "+ + sstableRecovery.candidates() + " temporary artifacts; "+ + sstableRecovery.inUse() + " still in use, "+ + sstableRecovery.failures() + " failures");+ }this.listsPath = this.getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT);this.log.config("Lists Path: " + this.listsPath.toString());this.htRootPath = this.getDataPath(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT); // path to the servlets@@ -770,8 +790,6 @@ public final class Switchboard extends serverSwitch {this.log.config("Starting HT Cache Manager");// create the cache directory- this.htCachePath =- this.getDataPath(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT);this.log.info("HTCACHE Path = " + this.htCachePath.getAbsolutePath());final long maxCacheSize =1024L * 1024L * Long.parseLong(this.getConfig(SwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabytediff --git a/test/java/net/yacy/cora/util/ChunkedBytesTest.java b/test/java/net/yacy/cora/util/ChunkedBytesTest.javanew file mode 100644index 000000000..1a6d79705--- /dev/null+++ b/test/java/net/yacy/cora/util/ChunkedBytesTest.java@@ -0,0 +1,232 @@+/*+ * ChunkedBytesTest+ * Copyright 2026 by Michael Peter Christen; [email protected], Frankfurt a. M., Germany+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ */++package net.yacy.cora.util;++import static org.junit.Assert.assertArrayEquals;+import static org.junit.Assert.assertEquals;+import static org.junit.Assert.assertFalse;+import static org.junit.Assert.assertThrows;+import static org.junit.Assert.assertTrue;++import java.io.File;+import java.io.IOException;+import java.io.InputStream;+import java.io.RandomAccessFile;+import java.io.UncheckedIOException;+import java.nio.file.Files;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;++import org.junit.Rule;+import org.junit.Test;+import org.junit.rules.TemporaryFolder;++public class ChunkedBytesTest {++ @Rule+ public final TemporaryFolder temporaryFolder = new TemporaryFolder();++ @Test+ public void inputStreamHonorsZeroLengthAndNegativeSkipContracts() throws Exception {+ try (ChunkedBytes empty = new ChunkedBytes();+ InputStream input = empty.openStream()) {+ assertEquals(0, input.read(new byte[1], 0, 0));+ }++ try (ChunkedBytes bytes = new ChunkedBytes(new byte[] {1, 2, 3});+ InputStream input = bytes.openStream()) {+ assertEquals(0L, input.skip(-1L));+ assertEquals(1, input.read());+ input.close();+ assertThrows(IOException.class, input::read);+ }+ }++ @Test+ public void validatesRangesBeforeReadingOrWriting() {+ try (ChunkedBytes bytes = new ChunkedBytes(new byte[] {1, 2, 3})) {+ assertEquals(0, bytes.read(bytes.size(), new byte[0], 0, 0));+ assertEquals(0, bytes.write(bytes.size(), new byte[0], 0, 0));+ assertThrows(NullPointerException.class,+ () -> bytes.read(bytes.size(), null, 0, 1));+ assertThrows(IndexOutOfBoundsException.class,+ () -> bytes.read(0, new byte[1], 1, 1));+ assertThrows(IndexOutOfBoundsException.class,+ () -> bytes.write(0, new byte[1], 0, -1));+ assertThrows(IndexOutOfBoundsException.class,+ () -> bytes.append(new byte[1], Integer.MAX_VALUE, 1));+ }+ }++ @Test+ public void equalContentHasEqualHashCodeAndExplicitForeignComparisons() {+ try (ChunkedBytes left = new ChunkedBytes(new byte[] {1, 2, 3});+ ChunkedBytes right = new ChunkedBytes(new byte[] {1, 2, 3})) {+ assertEquals(left, right);+ assertEquals(left.hashCode(), right.hashCode());+ assertFalse(left.equals(new byte[] {1, 2, 3}));+ assertFalse(left.equals("\u0001\u0002\u0003"));+ assertTrue(left.contentEquals(new byte[] {1, 2, 3}));+ assertTrue(left.contentEquals("\u0001\u0002\u0003"));+ }+ }++ @Test+ public void closeIsIdempotentAndRejectsFurtherAccess() {+ final ChunkedBytes bytes = new ChunkedBytes(new byte[] {1});+ bytes.close();+ bytes.close();++ assertThrows(IllegalStateException.class, bytes::size);+ assertThrows(IllegalStateException.class, () -> bytes.get(0));+ assertThrows(IllegalStateException.class, () -> bytes.append(new byte[] {2}));+ assertThrows(IllegalStateException.class, bytes::openStream);+ }++ @Test+ public void fileBackedCloneOutlivesOriginalOwner() throws Exception {+ final File file = this.temporaryFolder.newFile("clone.bin");+ Files.write(file.toPath(), new byte[] {42, 43});++ final ChunkedBytes original = new ChunkedBytes();+ original.appendFile(file.toPath());+ final ChunkedBytes clone = (ChunkedBytes) original.clone();+ original.close();+ try {+ assertEquals(42, clone.get(0));+ assertArrayEquals(new byte[] {42, 43}, clone.toByteArray());+ } finally {+ clone.close();+ }+ }++ @Test(timeout = 15000L)+ public void comparesDifferentContentBeyondTwoGiB() throws Exception {+ final long length = (long) Integer.MAX_VALUE + 1L;+ final File leftFile = this.temporaryFolder.newFile("large-left.bin");+ final File rightFile = this.temporaryFolder.newFile("large-right.bin");+ createSparseFile(leftFile, length, 1);+ createSparseFile(rightFile, length, 2);++ try (ChunkedBytes left = new ChunkedBytes();+ ChunkedBytes right = new ChunkedBytes()) {+ left.appendFile(leftFile.toPath());+ right.appendFile(rightFile.toPath());+ assertEquals(length, left.size());+ assertEquals(length, right.size());+ assertTrue(left.compareTo(right) < 0);+ assertTrue(right.compareTo(left) > 0);+ }+ }++ @Test+ public void writableMappingPersistsCrossChunkChangesOnFlushAndClose() throws Exception {+ final File file = this.temporaryFolder.newFile("writable.bin");+ final long patchPosition = ChunkedBytes.CHUNK_SIZE - 4L;+ final byte[] patch = new byte[] {9, 8, 7, 6, 5, 4, 3, 2};+ createSparseFile(file, ChunkedBytes.CHUNK_SIZE + 4L, 0);++ try (ChunkedBytes bytes = new ChunkedBytes()) {+ bytes.appendFile(file.toPath(), true);+ assertEquals(patch.length,+ bytes.write(patchPosition, patch, 0, patch.length));+ bytes.flush();+ }++ final byte[] stored = new byte[patch.length];+ try (RandomAccessFile input = new RandomAccessFile(file, "r")) {+ input.seek(patchPosition);+ input.readFully(stored);+ }+ assertArrayEquals(patch, stored);+ }++ @Test+ public void appendFileReportsIoFailuresConsistently() {+ final File missing = new File(this.temporaryFolder.getRoot(), "missing.bin");+ try (ChunkedBytes bytes = new ChunkedBytes()) {+ assertThrows(UncheckedIOException.class,+ () -> bytes.appendFile(missing.toPath()));+ }+ }++ @Test(timeout = 5000L)+ public void writeFromDoesNotHoldMonitorWhileCallingInputStream() throws Exception {+ final ChunkedBytes bytes = new ChunkedBytes();+ final Object inputLock = new Object();+ final CountDownLatch inputLockHeld = new CountDownLatch(1);+ final CountDownLatch readEntered = new CountDownLatch(1);+ final AtomicReference<Throwable> failure = new AtomicReference<>();++ final InputStream input = new InputStream() {+ @Override+ public int read() {+ throw new AssertionError("Bulk read expected");+ }++ @Override+ public int read(final byte[] target, final int offset, final int length) {+ readEntered.countDown();+ synchronized (inputLock) {+ return -1;+ }+ }+ };++ final Thread appender = new Thread(() -> {+ synchronized (inputLock) {+ inputLockHeld.countDown();+ try {+ if (!readEntered.await(2L, TimeUnit.SECONDS)) {+ throw new AssertionError("writeFrom did not enter InputStream.read");+ }+ bytes.append(new byte[] {1});+ } catch (final Throwable e) {+ failure.compareAndSet(null, e);+ }+ }+ }, "chunked-bytes-lock-owner");+ appender.setDaemon(true);+ appender.start();+ assertTrue("Input lock was not acquired",+ inputLockHeld.await(2L, TimeUnit.SECONDS));++ final Thread writer = new Thread(() -> {+ try {+ bytes.writeFrom(input);+ } catch (final Throwable e) {+ failure.compareAndSet(null, e);+ }+ }, "chunked-bytes-stream-writer");+ writer.setDaemon(true);+ writer.start();++ writer.join(2000L);+ appender.join(2000L);+ assertFalse("writeFrom and append deadlocked", writer.isAlive());+ assertFalse("writeFrom and append deadlocked", appender.isAlive());+ if (failure.get() != null) throw new AssertionError(failure.get());+ assertArrayEquals(new byte[] {1}, bytes.toByteArray());+ bytes.close();+ }++ private static void createSparseFile(final File file, final long length,+ final int firstByte) throws IOException {+ try (RandomAccessFile output = new RandomAccessFile(file, "rw")) {+ output.setLength(length);+ if (length > 0) {+ output.seek(0L);+ output.write(firstByte);+ }+ }+ }+}diff --git a/test/java/net/yacy/kelondro/blob/ArrayStackSSTableLifecycleTest.java b/test/java/net/yacy/kelondro/blob/ArrayStackSSTableLifecycleTest.javanew file mode 100644index 000000000..bfbb3607a--- /dev/null+++ b/test/java/net/yacy/kelondro/blob/ArrayStackSSTableLifecycleTest.java@@ -0,0 +1,308 @@+/*+ * ArrayStackSSTableLifecycleTest+ * Copyright 2026 by Michael Peter Christen; [email protected], Frankfurt a. M., Germany+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ */++package net.yacy.kelondro.blob;++import static org.junit.Assert.assertArrayEquals;+import static org.junit.Assert.assertEquals;+import static org.junit.Assert.assertFalse;+import static org.junit.Assert.assertNotNull;+import static org.junit.Assert.assertTrue;++import java.io.File;+import java.io.UncheckedIOException;+import java.lang.reflect.Field;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.Date;+import java.util.List;++import org.junit.Rule;+import org.junit.Test;+import org.junit.rules.TemporaryFolder;++import net.yacy.cora.document.encoding.ASCII;+import net.yacy.cora.order.NaturalOrder;+import net.yacy.kelondro.data.citation.CitationReference;+import net.yacy.kelondro.data.citation.CitationReferenceFactory;+import net.yacy.kelondro.index.RowHandleMap;+import net.yacy.kelondro.index.RowSet;+import net.yacy.kelondro.index.SSTableHandleMap;+import net.yacy.kelondro.rwi.ReferenceContainer;++public class ArrayStackSSTableLifecycleTest {++ private static final int KEY_LENGTH = 12;+ private static final int WRITE_BUFFER = 4096;+ private static final String PREFIX = "segment";++ private static final byte[] DELETE_KEY = ASCII.getBytes("AAAAAAAAAAAA");+ private static final byte[] FILLER_B_KEY = ASCII.getBytes("BBBBBBBBBBBB");+ private static final byte[] REDUCE_KEY = ASCII.getBytes("CCCCCCCCCCCC");+ private static final byte[] SHARED_KEY = ASCII.getBytes("DDDDDDDDDDDD");+ private static final byte[] FILLER_E_KEY = ASCII.getBytes("EEEEEEEEEEEE");+ private static final byte[] ACTIVE_KEY = ASCII.getBytes("FFFFFFFFFFFF");+ private static final byte[] FILLER_G_KEY = ASCII.getBytes("GGGGGGGGGGGG");+ private static final byte[] FILLER_H_KEY = ASCII.getBytes("HHHHHHHHHHHH");+ private static final byte[] FILLER_I_KEY = ASCII.getBytes("IIIIIIIIIIII");+ private static final byte[] RECOVERED_KEY = ASCII.getBytes("JJJJJJJJJJJJ");++ private static final CitationReferenceFactory REFERENCE_FACTORY =+ new CitationReferenceFactory();++ @Rule+ public final TemporaryFolder temporaryFolder = new TemporaryFolder();++ @Test+ public void sealedSegmentsStaySSTablesAcrossRestartReduceDeleteAndMerge() throws Exception {+ final File directory = this.temporaryFolder.newFolder("array-stack-sstable");+ ArrayStack stack = newStack(directory);+ try {+ final byte[] twoReferences = container(+ REDUCE_KEY, "000000000001", "000000000002");+ final byte[] oneReference = container(REDUCE_KEY, "000000000001");++ /*+ * The first five inserts create the initial writable Heap. Five entries+ * are intentional: Heap.close(true) then publishes an idx/gap pair which+ * sealWriter() must reopen through HeapReader.immutable().+ */+ stack.insert(DELETE_KEY, container(DELETE_KEY, "100000000001"));+ stack.insert(FILLER_B_KEY, container(FILLER_B_KEY, "200000000001"));+ stack.insert(REDUCE_KEY, twoReferences);+ stack.insert(SHARED_KEY, container(SHARED_KEY, "300000000001"));+ stack.insert(FILLER_E_KEY, container(FILLER_E_KEY, "400000000001"));++ /*+ * Mounting a newer full segment is the deterministic rollover path:+ * ArrayStack seals the old writer and keeps only this new segment mutable.+ */+ final File secondSegment = stack.newBLOB(+ new Date(System.currentTimeMillis() + 60_000L));+ writeSecondSegment(secondSegment);+ stack.mountBLOB(secondSegment, true);++ assertSegmentTypes(stack, HeapModifier.class, Heap.class);+ assertTrue(((HeapModifier) segmentBlobs(stack).get(0)).index+ instanceof SSTableHandleMap);+ assertTrue(((Heap) segmentBlobs(stack).get(1)).index+ instanceof RowHandleMap);+ assertEquals(0L, countWorkingFiles(directory.toPath()));+ assertArrayEquals(container(SHARED_KEY, "500000000001"),+ stack.get(SHARED_KEY));+ assertEquals(2, countOccurrences(stack, SHARED_KEY));++ /* reduce() must use HeapModifier's in-place path on the sealed segment. */+ assertEquals(twoReferences.length - oneReference.length,+ stack.reduce(REDUCE_KEY, ignored -> oneReference));+ assertEquals(1L, countWorkingFiles(directory.toPath()));+ assertContainerSize(stack.get(REDUCE_KEY), 1);++ stack.close(true);+ stack = null;+ assertEquals(0L, countWorkingFiles(directory.toPath()));++ /* A process restart must reconstruct the same mutable/sealed boundary. */+ stack = newStack(directory);+ assertSegmentTypes(stack, HeapModifier.class, Heap.class);+ assertTrue(((HeapModifier) segmentBlobs(stack).get(0)).index+ instanceof SSTableHandleMap);+ assertTrue(((Heap) segmentBlobs(stack).get(1)).index+ instanceof RowHandleMap);+ assertContainerSize(stack.get(REDUCE_KEY), 1);+ assertEquals(0L, countWorkingFiles(directory.toPath()));++ /*+ * Exercise the actual ArrayStack merge protocol: unmount both sources,+ * merge their ReferenceContainers, and mount the result as a sealed heap.+ */+ final File[] mergeSources = stack.unmountSmallest(Long.MAX_VALUE);+ assertNotNull(mergeSources);+ assertEquals(0, stack.entries());+ assertEquals(0L, countWorkingFiles(directory.toPath()));++ final File mergedFile = stack.newBLOB(+ new Date(System.currentTimeMillis() + 120_000L));+ assertEquals(mergedFile, stack.mergeMount(+ mergeSources[0], mergeSources[1], REFERENCE_FACTORY,+ mergedFile, WRITE_BUFFER));+ assertSegmentTypes(stack, HeapModifier.class);+ assertTrue(((HeapModifier) segmentBlobs(stack).get(0)).index+ instanceof SSTableHandleMap);+ assertEquals(0L, countWorkingFiles(directory.toPath()));+ assertContainerSize(stack.get(SHARED_KEY), 2);+ assertContainerSize(stack.get(REDUCE_KEY), 1);+ assertEquals(9, stack.size());++ /* With one sealed segment delete() reaches HeapModifier without a RAM index. */+ stack.delete(DELETE_KEY);+ assertEquals(1L, countWorkingFiles(directory.toPath()));+ assertFalse(stack.containsKey(DELETE_KEY));+ assertEquals(8, stack.size());++ stack.close(true);+ stack = null;+ assertEquals(0L, countWorkingFiles(directory.toPath()));++ /* The newest (now sole) merged segment is writable after the next restart. */+ stack = newStack(directory);+ assertSegmentTypes(stack, Heap.class);+ assertTrue(((Heap) segmentBlobs(stack).get(0)).index+ instanceof RowHandleMap);+ assertFalse(stack.containsKey(DELETE_KEY));+ assertContainerSize(stack.get(SHARED_KEY), 2);+ assertContainerSize(stack.get(REDUCE_KEY), 1);+ assertEquals(8, stack.size());+ assertEquals(0L, countWorkingFiles(directory.toPath()));+ } finally {+ if (stack != null) stack.close(false);+ }+ }++ @Test+ public void failedFingerprintPublishRestoresWritableHeap() throws Exception {+ final File directory = this.temporaryFolder.newFolder("array-stack-publish-failure");+ ArrayStack stack = new ArrayStack(+ directory, PREFIX, NaturalOrder.naturalOrder,+ KEY_LENGTH, 0, false, false);+ try {+ /* More than three entries make close(true) publish a fingerprint pair. */+ stack.insert(DELETE_KEY, ASCII.getBytes("value-a"));+ stack.insert(FILLER_B_KEY, ASCII.getBytes("value-b"));+ stack.insert(REDUCE_KEY, ASCII.getBytes("value-c"));+ stack.insert(SHARED_KEY, ASCII.getBytes("value-d"));+ stack.insert(FILLER_E_KEY, ASCII.getBytes("value-e"));++ final File activeBlob = onlyBlobFile(directory);+ final String fingerprint = HeapReader.fingerprintFileHash(activeBlob);+ final File gapTarget = HeapWriter.fingerprintGapFile(activeBlob, fingerprint);++ /*+ * A non-empty directory cannot be replaced by Gap.dump(). This forces+ * publishing to fail after Heap.close() has already released its file+ * and in-memory index, reproducing the stale-writer failure mode.+ */+ assertTrue(gapTarget.mkdir());+ Files.write(gapTarget.toPath().resolve("blocker"), new byte[] {1});++ final File futureSegment = stack.newBLOB(+ new Date(System.currentTimeMillis() + 60_000L));+ UncheckedIOException publishFailure = null;+ try {+ stack.mountBLOB(futureSegment, true);+ } catch (final UncheckedIOException expected) {+ publishFailure = expected;+ }+ assertNotNull("The forced fingerprint publish must fail", publishFailure);++ /* The original segment must already be reopened and writable. */+ final byte[] recoveredValue = ASCII.getBytes("after-failure");+ stack.insert(RECOVERED_KEY, recoveredValue);+ assertArrayEquals(recoveredValue, stack.get(RECOVERED_KEY));++ stack.close(true);+ stack = null;++ stack = new ArrayStack(+ directory, PREFIX, NaturalOrder.naturalOrder,+ KEY_LENGTH, 0, false, false);+ assertArrayEquals(recoveredValue, stack.get(RECOVERED_KEY));+ } finally {+ if (stack != null) stack.close(false);+ }+ }++ private static ArrayStack newStack(final File directory) throws Exception {+ return new ArrayStack(directory, PREFIX, NaturalOrder.naturalOrder,+ KEY_LENGTH, WRITE_BUFFER, false, false);+ }++ private static File onlyBlobFile(final File directory) {+ final File[] blobs = directory.listFiles(+ file -> file.getName().endsWith(".blob"));+ assertNotNull(blobs);+ assertEquals(1, blobs.length);+ return blobs[0];+ }++ private static void writeSecondSegment(final File target) throws Exception {+ final File temporary = new File(target.getParentFile(), target.getName() + ".prt");+ final HeapWriter writer = new HeapWriter(+ temporary, target, KEY_LENGTH, NaturalOrder.naturalOrder, WRITE_BUFFER);+ writer.add(SHARED_KEY, container(SHARED_KEY, "500000000001"));+ writer.add(ACTIVE_KEY, container(ACTIVE_KEY, "600000000001"));+ writer.add(FILLER_G_KEY, container(FILLER_G_KEY, "700000000001"));+ writer.add(FILLER_H_KEY, container(FILLER_H_KEY, "800000000001"));+ writer.add(FILLER_I_KEY, container(FILLER_I_KEY, "900000000001"));+ writer.close(true);+ }++ private static byte[] container(final byte[] termHash, final String... urlHashes)+ throws Exception {+ final ReferenceContainer<CitationReference> container =+ new ReferenceContainer<>(REFERENCE_FACTORY, termHash);+ long modified = 1_700_000_000_000L;+ for (final String urlHash : urlHashes) {+ container.add(new CitationReference(ASCII.getBytes(urlHash), modified++));+ }+ return container.exportCollection();+ }++ private static void assertContainerSize(final byte[] payload, final int expected)+ throws Exception {+ assertNotNull(payload);+ assertEquals(expected,+ RowSet.importRowSet(payload, REFERENCE_FACTORY.getRow()).size());+ }++ private static int countOccurrences(final ArrayStack stack, final byte[] key)+ throws Exception {+ int count = 0;+ for (final byte[] payload : stack.getAll(key)) {+ assertContainerSize(payload, 1);+ count++;+ }+ return count;+ }++ private static void assertSegmentTypes(+ final ArrayStack stack, final Class<?>... expectedTypes) throws Exception {+ final List<ImmutableBLOB> segments = segmentBlobs(stack);+ assertEquals(expectedTypes.length, segments.size());+ for (int index = 0; index < expectedTypes.length; index++) {+ assertEquals(expectedTypes[index], segments.get(index).getClass());+ }+ }++ /**+ * ArrayStack deliberately keeps blobItem private. Reflection is limited to+ * this regression assertion; production code still exposes no mutable segment list.+ */+ private static List<ImmutableBLOB> segmentBlobs(final ArrayStack stack) throws Exception {+ final Field blobsField = ArrayStack.class.getDeclaredField("blobs");+ blobsField.setAccessible(true);+ final List<?> items = (List<?>) blobsField.get(stack);+ final List<ImmutableBLOB> result = new ArrayList<>(items.size());+ for (final Object item : items) {+ final Field blobField = item.getClass().getDeclaredField("blob");+ blobField.setAccessible(true);+ result.add((ImmutableBLOB) blobField.get(item));+ }+ return result;+ }++ private static long countWorkingFiles(final Path directory) throws Exception {+ try (java.util.stream.Stream<Path> files = Files.list(directory)) {+ return files.filter(path -> path.getFileName().toString()+ .endsWith(".sstwork")).count();+ }+ }+}diff --git a/test/java/net/yacy/kelondro/blob/HeapCloseFailureTest.java b/test/java/net/yacy/kelondro/blob/HeapCloseFailureTest.javanew file mode 100644index 000000000..43e2b2199--- /dev/null+++ b/test/java/net/yacy/kelondro/blob/HeapCloseFailureTest.java@@ -0,0 +1,104 @@+/*+ * HeapCloseFailureTest+ * Copyright 2026 by Michael Peter Christen; [email protected], Frankfurt a. M., Germany+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ */++package net.yacy.kelondro.blob;++import static org.junit.Assert.assertEquals;+import static org.junit.Assert.assertFalse;+import static org.junit.Assert.assertTrue;++import java.io.File;+import java.io.IOException;+import java.io.UncheckedIOException;+import java.lang.reflect.InvocationTargetException;+import java.lang.reflect.Proxy;++import org.junit.Rule;+import org.junit.Test;+import org.junit.rules.TemporaryFolder;++import net.yacy.cora.document.encoding.ASCII;+import net.yacy.cora.order.NaturalOrder;+import net.yacy.kelondro.io.Writer;++public class HeapCloseFailureTest {++ private static final int KEY_LENGTH = 12;++ @Rule+ public final TemporaryFolder temporaryFolder = new TemporaryFolder();++ @Test+ public void closePropagatesFlushFailureWithoutPublishingStaleIndex() throws Exception {+ final File directory = this.temporaryFolder.newFolder("heap-close-failure");+ final File heapFile = new File(directory, "data.blob");+ final Heap heap = new Heap(+ heapFile, KEY_LENGTH, NaturalOrder.naturalOrder, 4096);++ for (int index = 0; index < 5; index++) {+ heap.insert(key(index), ASCII.getBytes("buffered-value-" + index));+ }++ /*+ * Keep length(), seek() and close() operational, but fail the bulk write.+ * At that point flushBufferInternal() has already assigned offsets to all+ * buffered keys, so close(true) must not publish this stale RAM index.+ */+ final Writer delegate = heap.file;+ heap.file = (Writer) Proxy.newProxyInstance(+ Writer.class.getClassLoader(),+ new Class<?>[] {Writer.class},+ (proxy, method, arguments) -> {+ if (method.getName().equals("write")) {+ throw new IOException("forced buffered write failure");+ }+ try {+ return method.invoke(delegate, arguments);+ } catch (final InvocationTargetException failure) {+ throw failure.getCause();+ }+ });++ try {+ heap.close(true);+ throw new AssertionError("flush failure was swallowed");+ } catch (final UncheckedIOException expected) {+ assertTrue(expected.getMessage().contains(heapFile.getName()));+ assertEquals("forced buffered write failure",+ expected.getCause().getMessage());+ } finally {+ heap.close(false);+ }++ assertFalse(hasFingerprintFile(directory));++ /* No stale offsets may survive: reopening must scan the authoritative blob. */+ final Heap reopened = new Heap(+ heapFile, KEY_LENGTH, NaturalOrder.naturalOrder, 4096);+ try {+ assertEquals(0, reopened.size());+ } finally {+ reopened.close(false);+ }+ }++ private static byte[] key(final int index) {+ return ASCII.getBytes(String.format("%012d", Integer.valueOf(index)));+ }++ private static boolean hasFingerprintFile(final File directory) {+ final File[] fingerprints = directory.listFiles(file -> {+ final String name = file.getName();+ return name.endsWith(".idx") || name.endsWith(".gap")+ || name.endsWith(".idx.gz") || name.endsWith(".gap.gz");+ });+ return fingerprints != null && fingerprints.length > 0;+ }+}diff --git a/test/java/net/yacy/kelondro/blob/HeapModifierWorkingCopyTest.java b/test/java/net/yacy/kelondro/blob/HeapModifierWorkingCopyTest.javanew file mode 100644index 000000000..70feab6ca--- /dev/null+++ b/test/java/net/yacy/kelondro/blob/HeapModifierWorkingCopyTest.java@@ -0,0 +1,208 @@+/*+ * HeapModifierWorkingCopyTest+ * Copyright 2026 by Michael Peter Christen; [email protected], Frankfurt a. M., Germany+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ */++package net.yacy.kelondro.blob;++import static org.junit.Assert.assertArrayEquals;+import static org.junit.Assert.assertEquals;+import static org.junit.Assert.assertFalse;+import static org.junit.Assert.assertTrue;++import java.io.File;+import java.io.UncheckedIOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;++import org.junit.Rule;+import org.junit.Test;+import org.junit.rules.TemporaryFolder;++import net.yacy.cora.order.NaturalOrder;+import net.yacy.kelondro.index.SSTableHandleMap;++public class HeapModifierWorkingCopyTest {++ @Rule+ public final TemporaryFolder temporaryFolder = new TemporaryFolder();++ @Test+ public void modifierUsesPrivateGenerationAndPublishesCompactedFingerprint() throws Exception {+ final File directory = this.temporaryFolder.newFolder("heap-working-copy");+ final File temporaryHeap = new File(directory, "data.heap.prt");+ final File heap = new File(directory, "data.heap");++ /* HeapWriter publishes the initial immutable fingerprint generation. */+ final HeapWriter writer = new HeapWriter(+ temporaryHeap, heap, 4, NaturalOrder.naturalOrder, 4096);+ for (int index = 1; index <= 5; index++) {+ writer.add(key(index), value(index));+ }+ writer.close(true);++ final File initialFingerprint = HeapWriter.fingerprintIndexFile(+ heap, HeapReader.fingerprintFileHash(heap));+ assertTrue(initialFingerprint.isFile());+ final byte[] publishedBytes = Files.readAllBytes(initialFingerprint.toPath());++ final HeapModifier first = new HeapModifier(+ heap, 4, NaturalOrder.naturalOrder);+ HeapModifier second = null;+ try {+ assertTrue(first.index instanceof SSTableHandleMap);+ assertEquals(0, countWorkingFiles(directory.toPath()));+ assertArrayEquals(value(2), first.get(key(2)));+ assertArrayEquals(publishedBytes, Files.readAllBytes(initialFingerprint.toPath()));++ /* Multiple readers remain on the published generation without copying. */+ second = new HeapModifier(heap, 4, NaturalOrder.naturalOrder);+ assertTrue(second.index instanceof SSTableHandleMap);+ assertEquals(0, countWorkingFiles(directory.toPath()));+ assertArrayEquals(value(2), second.get(key(2)));+ second.close(false);+ second = null;+ assertEquals(0, countWorkingFiles(directory.toPath()));++ /* The first heap mutation creates exactly one private generation. */+ first.delete(key(2));+ assertEquals(1, countWorkingFiles(directory.toPath()));+ assertFalse(first.containsKey(key(2)));+ assertFalse(initialFingerprint.exists());+ first.close(true);+ } finally {+ if (second != null) second.close(false);+ first.close(false);+ }++ assertEquals(0, countWorkingFiles(directory.toPath()));++ /* close(true) compacted tombstones and published the next fingerprint. */+ final File compactedFingerprint = HeapWriter.fingerprintIndexFile(+ heap, HeapReader.fingerprintFileHash(heap));+ assertTrue(compactedFingerprint.isFile());++ final HeapModifier reopened = new HeapModifier(+ heap, 4, NaturalOrder.naturalOrder);+ try {+ assertTrue(reopened.index instanceof SSTableHandleMap);+ assertEquals(4, reopened.size());+ assertFalse(reopened.containsKey(key(2)));+ assertArrayEquals(value(5), reopened.get(key(5)));+ } finally {+ reopened.close(true);+ }+ assertEquals(0, countWorkingFiles(directory.toPath()));+ }++ @Test+ public void publishFailureIsPropagatedAndRollsBackIncompletePair() throws Exception {+ final File directory = this.temporaryFolder.newFolder("heap-publish-failure");+ final File temporaryHeap = new File(directory, "data.heap.prt");+ final File heap = new File(directory, "data.heap");++ final HeapWriter writer = new HeapWriter(+ temporaryHeap, heap, 4, NaturalOrder.naturalOrder, 4096);+ for (int index = 1; index <= 5; index++) {+ writer.add(key(index), value(index));+ }+ writer.close(true);++ final HeapModifier modifier = new HeapModifier(+ heap, 4, NaturalOrder.naturalOrder);+ modifier.delete(key(2));+ final String fingerprint = HeapReader.fingerprintFileHash(heap);+ final File targetIndex = HeapWriter.fingerprintIndexFile(heap, fingerprint);+ final File targetGap = HeapWriter.fingerprintGapFile(heap, fingerprint);++ /* A non-empty directory cannot be replaced by the final gap dump. The+ * index has already been published at that point and must be rolled back. */+ assertTrue(targetGap.mkdir());+ Files.write(targetGap.toPath().resolve("blocker"), new byte[] {1});++ try {+ modifier.close(true);+ throw new AssertionError("publish failure was swallowed");+ } catch (final UncheckedIOException expected) {+ assertTrue(expected.getMessage().contains(heap.getName()));+ assertTrue(expected.getCause().getMessage().contains("complete fingerprint pair"));+ } finally {+ modifier.close(false);+ }++ assertFalse(targetIndex.exists());+ assertTrue(targetGap.isDirectory());+ assertFalse(new File(targetIndex.getPath() + ".prt").exists());+ assertFalse(new File(targetGap.getPath() + ".prt").exists());+ assertEquals(0, countWorkingFiles(directory.toPath()));+ }++ @Test+ public void incompleteFingerprintPairIsRebuiltFromHeapAndRepublished() throws Exception {+ final File directory = this.temporaryFolder.newFolder("heap-pair-recovery");+ final File temporaryHeap = new File(directory, "data.heap.prt");+ final File heap = new File(directory, "data.heap");++ final HeapWriter writer = new HeapWriter(+ temporaryHeap, heap, 4, NaturalOrder.naturalOrder, 4096);+ for (int index = 1; index <= 6; index++) {+ writer.add(key(index), value(index));+ }+ writer.close(true);++ final String fingerprint = HeapReader.fingerprintFileHash(heap);+ final File indexDump = HeapWriter.fingerprintIndexFile(heap, fingerprint);+ final File gapDump = HeapWriter.fingerprintGapFile(heap, fingerprint);+ assertTrue(indexDump.isFile());+ assertTrue(gapDump.isFile());++ /* Simulate a crash between publishing the index and installing the gap+ * commit marker. The remaining index must never be accepted alone. */+ Files.delete(gapDump.toPath());+ assertFalse(gapDump.exists());++ final HeapModifier recovered = new HeapModifier(+ heap, 4, NaturalOrder.naturalOrder);+ try {+ assertEquals(6, recovered.size());+ assertArrayEquals(value(1), recovered.get(key(1)));+ assertArrayEquals(value(6), recovered.get(key(6)));+ assertFalse(indexDump.exists());+ } finally {+ recovered.close(true);+ }++ assertTrue(indexDump.isFile());+ assertTrue(gapDump.isFile());+ final HeapModifier restarted = new HeapModifier(+ heap, 4, NaturalOrder.naturalOrder);+ try {+ assertEquals(6, restarted.size());+ assertArrayEquals(value(4), restarted.get(key(4)));+ } finally {+ restarted.close(false);+ }+ assertEquals(0, countWorkingFiles(directory.toPath()));+ }++ private static long countWorkingFiles(final Path directory) throws Exception {+ try (java.util.stream.Stream<Path> files = Files.list(directory)) {+ return files.filter(path -> path.getFileName().toString()+ .endsWith(".sstwork")).count();+ }+ }++ private static byte[] key(final int value) {+ return String.format("%04d", value).getBytes(StandardCharsets.US_ASCII);+ }++ private static byte[] value(final int value) {+ return ("value-" + value).getBytes(StandardCharsets.US_ASCII);+ }+}diff --git a/test/java/net/yacy/kelondro/blob/HeapReaderInitializationFailureTest.java b/test/java/net/yacy/kelondro/blob/HeapReaderInitializationFailureTest.javanew file mode 100644index 000000000..075d3ecec--- /dev/null+++ b/test/java/net/yacy/kelondro/blob/HeapReaderInitializationFailureTest.java@@ -0,0 +1,141 @@+/*+ * HeapReaderInitializationFailureTest+ * Copyright 2026 by Michael Peter Christen; [email protected], Frankfurt a. M., Germany+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ */++package net.yacy.kelondro.blob;++import static org.junit.Assert.assertArrayEquals;+import static org.junit.Assert.assertEquals;+import static org.junit.Assert.assertTrue;++import java.io.BufferedOutputStream;+import java.io.DataOutputStream;+import java.io.File;+import java.io.IOException;+import java.lang.reflect.InvocationTargetException;+import java.lang.reflect.Proxy;+import java.nio.file.Files;+import java.util.concurrent.atomic.AtomicBoolean;++import org.junit.Rule;+import org.junit.Test;+import org.junit.rules.TemporaryFolder;++import net.yacy.cora.document.encoding.ASCII;+import net.yacy.cora.order.ByteOrder;+import net.yacy.cora.order.NaturalOrder;+import net.yacy.cora.storage.HandleMap;+import net.yacy.kelondro.io.CachedFileWriter;+import net.yacy.kelondro.io.Writer;++public class HeapReaderInitializationFailureTest {++ private static final int KEY_LENGTH = 4;+ private static final byte[] KEY = ASCII.getBytes("0001");+ private static final byte[] VALUE = ASCII.getBytes("payload");++ @Rule+ public final TemporaryFolder temporaryFolder = new TemporaryFolder();++ @Test+ public void rebuildFailureClosesBuilderAndHeapFile() throws Exception {+ final File directory = this.temporaryFolder.newFolder("failed-rebuild");+ final File heapFile = new File(directory, "data.blob");+ writeHeapRecord(heapFile);++ final AtomicBoolean fileClosed = new AtomicBoolean();+ final AtomicBoolean builderClosed = new AtomicBoolean();+ final HeapReader.HeapFileFactory files = file -> {+ final Writer delegate = new CachedFileWriter(file);+ return (Writer) Proxy.newProxyInstance(+ Writer.class.getClassLoader(),+ new Class<?>[] {Writer.class},+ (proxy, method, arguments) -> {+ if (method.getName().equals("close")) fileClosed.set(true);+ try {+ return method.invoke(delegate, arguments);+ } catch (final InvocationTargetException failure) {+ throw failure.getCause();+ }+ });+ };++ final HeapReader.HeapIndexFactory<HandleMap> indexes =+ new HeapReader.HeapIndexFactory<HandleMap>() {+ @Override+ public HandleMap load(final int keylength, final ByteOrder ordering,+ final File dump) {+ throw new AssertionError("No fingerprint should be loaded");+ }++ @Override+ public HeapReader.HeapIndexBuilder<HandleMap> newBuilder(+ final File file, final int keylength,+ final ByteOrder ordering, final int expectedspace) {+ return new HeapReader.HeapIndexBuilder<HandleMap>() {+ @Override+ public void consume(final byte[] key, final long offset)+ throws IOException {+ throw new IOException("forced index rebuild failure");+ }++ @Override+ public HandleMap finish() throws IOException {+ throw new IOException("finish must not be reached");+ }++ @Override+ public void close() {+ builderClosed.set(true);+ }+ };+ }++ @Override+ public void prepareForHeapMutation(final HandleMap index) {+ }+ };++ try {+ new TestHeapReader(heapFile, indexes, files);+ throw new AssertionError("index rebuild failure was swallowed");+ } catch (final IOException expected) {+ assertEquals("forced index rebuild failure", expected.getMessage());+ }+ assertTrue("The failed builder must be closed", builderClosed.get());+ assertTrue("The heap file must be closed", fileClosed.get());++ /* Cleanup must leave the authoritative heap usable for a normal retry. */+ final Heap recovered = new Heap(+ heapFile, KEY_LENGTH, NaturalOrder.naturalOrder, 0);+ try {+ assertArrayEquals(VALUE, recovered.get(KEY));+ } finally {+ recovered.close(false);+ }+ }++ private static void writeHeapRecord(final File heapFile) throws IOException {+ try (DataOutputStream output = new DataOutputStream(+ new BufferedOutputStream(Files.newOutputStream(heapFile.toPath())))) {+ output.writeInt(KEY_LENGTH + VALUE.length);+ output.write(KEY);+ output.write(VALUE);+ }+ }++ private static final class TestHeapReader extends HeapReader<HandleMap> {+ private TestHeapReader(final File heapFile,+ final HeapIndexFactory<HandleMap> indexFactory,+ final HeapFileFactory heapFileFactory) throws IOException {+ super(heapFile, KEY_LENGTH, NaturalOrder.naturalOrder,+ indexFactory, heapFileFactory);+ }+ }+}diff --git a/test/java/net/yacy/kelondro/index/SSTableHandleMapRecoveryAndLoadTest.java b/test/java/net/yacy/kelondro/index/SSTableHandleMapRecoveryAndLoadTest.javanew file mode 100644index 000000000..7af87bea7--- /dev/null+++ b/test/java/net/yacy/kelondro/index/SSTableHandleMapRecoveryAndLoadTest.java@@ -0,0 +1,280 @@+/*+ * SSTableHandleMapRecoveryAndLoadTest+ * Copyright 2026 by Michael Peter Christen; [email protected], Frankfurt a. M., Germany+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ */++package net.yacy.kelondro.index;++import static org.junit.Assert.assertArrayEquals;+import static org.junit.Assert.assertEquals;+import static org.junit.Assert.assertTrue;++import java.io.BufferedOutputStream;+import java.io.BufferedReader;+import java.io.DataOutputStream;+import java.io.File;+import java.io.IOException;+import java.io.InputStreamReader;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.List;+import java.util.Locale;+import java.util.Random;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;++import org.junit.Rule;+import org.junit.Test;+import org.junit.rules.TemporaryFolder;++import net.yacy.cora.order.NaturalOrder;++public class SSTableHandleMapRecoveryAndLoadTest {++ private static final int KEY_LENGTH = Long.BYTES;+ private static final int LARGE_RECORD_COUNT = 200000;+ private static final int LOOKUPS_PER_READER = 20000;+ private static final int REMOVALS = 5000;++ @Rule+ public final TemporaryFolder temporaryFolder = new TemporaryFolder();++ /**+ * Helper entry point used by {@link #globalRecoveryRemovesARealCrashArtifact()}.+ * The process deliberately bypasses every shutdown hook and finally block.+ */+ public static void main(final String[] arguments) throws Exception {+ if (arguments.length != 2) return;+ if ("--crash-helper".equals(arguments[0])) {+ final File fingerprint = new File(arguments[1]);+ final SSTableHandleMap table = SSTableHandleMap.openWorkingCopy(+ KEY_LENGTH, NaturalOrder.naturalOrder, Long.BYTES, fingerprint);+ table.remove(key(1));+ signalReadyAndWait();+ /* Keep the owning table strongly reachable while the parent verifies its+ * cross-process lock. The helper is normally killed before this executes. */+ table.size();+ Runtime.getRuntime().halt(0);+ }+ if ("--builder-crash-helper".equals(arguments[0])) {+ final SSTableHandleMap.Builder builder = new SSTableHandleMap.Builder(+ KEY_LENGTH, NaturalOrder.naturalOrder, Long.BYTES,+ new File(arguments[1]), "crash-helper", 1);+ builder.consume(key(1), offset(1));+ signalReadyAndWait();+ /* Preserve the Builder and its session lock until the forced process stop. */+ builder.consume(key(2), offset(2));+ Runtime.getRuntime().halt(0);+ }+ }++ private static void signalReadyAndWait() throws IOException {+ System.out.println("READY");+ System.out.flush();+ System.in.read();+ }++ @Test(timeout = 30000)+ public void globalRecoveryRemovesARealCrashArtifact() throws Exception {+ final File repository = this.temporaryFolder.newFolder("crash-repository");+ final File fingerprint = new File(repository, "crashed.idx");+ writeTable(fingerprint, 32);+ final byte[] published = Files.readAllBytes(fingerprint.toPath());++ final File java = new File(new File(System.getProperty("java.home"), "bin"),+ isWindows() ? "java.exe" : "java");+ final Process process = new ProcessBuilder(+ java.getAbsolutePath(), "-cp", System.getProperty("java.class.path"),+ SSTableHandleMapRecoveryAndLoadTest.class.getName(),+ "--crash-helper", fingerprint.getAbsolutePath())+ .redirectErrorStream(true)+ .start();+ final BufferedReader processOutput = new BufferedReader(new InputStreamReader(+ process.getInputStream(), StandardCharsets.UTF_8));+ try {+ /* Newer JVMs may print runtime diagnostics (for example about explicit+ * mapped-buffer cleanup) before the helper reaches its synchronization+ * marker. These messages must not be mistaken for helper failure. */+ String outputLine;+ do {+ outputLine = processOutput.readLine();+ } while (outputLine != null && !"READY".equals(outputLine));+ assertEquals("READY", outputLine);+ assertEquals(1L, countWorkingFiles(repository.toPath()));++ final SSTableHandleMap.RecoveryReport whileRunning =+ SSTableHandleMap.recoverWorkingCopies(repository);+ assertEquals(1L, whileRunning.candidates());+ assertEquals(0L, whileRunning.deleted());+ assertEquals(1L, whileRunning.inUse());+ assertEquals(0L, whileRunning.failures());+ } finally {+ process.destroyForcibly();+ if (!process.waitFor(20, TimeUnit.SECONDS)) {+ process.destroyForcibly();+ throw new AssertionError("crash helper did not terminate");+ }+ }+ assertEquals(1L, countWorkingFiles(repository.toPath()));++ final SSTableHandleMap.RecoveryReport recovery =+ SSTableHandleMap.recoverWorkingCopies(repository);+ assertEquals(1L, recovery.candidates());+ assertEquals(1L, recovery.deleted());+ assertEquals(0L, recovery.inUse());+ assertEquals(0L, recovery.failures());+ assertEquals(0L, countWorkingFiles(repository.toPath()));+ assertArrayEquals(published, Files.readAllBytes(fingerprint.toPath()));+ }++ @Test(timeout = 30000)+ public void globalRecoveryRemovesRealBuilderCrashArtifacts() throws Exception {+ final File repository = this.temporaryFolder.newFolder("builder-crash-repository");+ final File java = new File(new File(System.getProperty("java.home"), "bin"),+ isWindows() ? "java.exe" : "java");+ final Process process = new ProcessBuilder(+ java.getAbsolutePath(), "-cp", System.getProperty("java.class.path"),+ SSTableHandleMapRecoveryAndLoadTest.class.getName(),+ "--builder-crash-helper", repository.getAbsolutePath())+ .redirectErrorStream(true)+ .start();+ final BufferedReader processOutput = new BufferedReader(new InputStreamReader(+ process.getInputStream(), StandardCharsets.UTF_8));+ try {+ String outputLine;+ do {+ outputLine = processOutput.readLine();+ } while (outputLine != null && !"READY".equals(outputLine));+ assertEquals("READY", outputLine);+ assertEquals(2L, countBuilderArtifacts(repository.toPath()));++ final SSTableHandleMap.RecoveryReport whileRunning =+ SSTableHandleMap.recoverWorkingCopies(repository);+ assertEquals(2L, whileRunning.candidates());+ assertEquals(0L, whileRunning.deleted());+ assertEquals(2L, whileRunning.inUse());+ assertEquals(0L, whileRunning.failures());+ } finally {+ process.destroyForcibly();+ if (!process.waitFor(20, TimeUnit.SECONDS)) {+ process.destroyForcibly();+ throw new AssertionError("builder crash helper did not terminate");+ }+ }++ final SSTableHandleMap.RecoveryReport recovery =+ SSTableHandleMap.recoverWorkingCopies(repository);+ assertEquals(2L, recovery.candidates());+ assertEquals(2L, recovery.deleted());+ assertEquals(0L, recovery.inUse());+ assertEquals(0L, recovery.failures());+ assertEquals(0L, countBuilderArtifacts(repository.toPath()));+ }++ @Test(timeout = 60000)+ public void largeTablePromotesSafelyUnderConcurrentReadLoad() throws Exception {+ final File repository = this.temporaryFolder.newFolder("load-repository");+ final File fingerprint = new File(repository, "large.idx");+ writeTable(fingerprint, LARGE_RECORD_COUNT);+ final byte[] published = Files.readAllBytes(fingerprint.toPath());++ final SSTableHandleMap table = SSTableHandleMap.openWorkingCopy(+ KEY_LENGTH, NaturalOrder.naturalOrder, Long.BYTES, fingerprint);+ final ExecutorService readers = Executors.newFixedThreadPool(4);+ try {+ assertEquals(LARGE_RECORD_COUNT, table.size());+ assertEquals(0L, table.mem());+ assertEquals(0L, countWorkingFiles(repository.toPath()));++ final CountDownLatch start = new CountDownLatch(1);+ final List<Future<?>> results = new ArrayList<>();+ for (int reader = 0; reader < 4; reader++) {+ final int seed = reader;+ results.add(readers.submit(() -> {+ final Random random = new Random(seed);+ start.await();+ for (int lookup = 0; lookup < LOOKUPS_PER_READER; lookup++) {+ /* Readers stay in the untouched upper half while the lower+ * half triggers and exercises the mapping transition. */+ final int record = LARGE_RECORD_COUNT / 2+ + random.nextInt(LARGE_RECORD_COUNT / 2);+ assertEquals(offset(record), table.get(key(record)));+ }+ return null;+ }));+ }++ start.countDown();+ for (int record = 0; record < REMOVALS; record++) {+ assertEquals(offset(record), table.remove(key(record)));+ }+ for (final Future<?> result : results) result.get();++ assertEquals(LARGE_RECORD_COUNT - REMOVALS, table.size());+ assertEquals(-1L, table.get(key(REMOVALS - 1)));+ assertEquals(offset(LARGE_RECORD_COUNT - 1),+ table.get(key(LARGE_RECORD_COUNT - 1)));+ assertEquals(1L, countWorkingFiles(repository.toPath()));+ assertArrayEquals(published, Files.readAllBytes(fingerprint.toPath()));+ } finally {+ readers.shutdownNow();+ try {+ assertTrue(readers.awaitTermination(10, TimeUnit.SECONDS));+ } finally {+ table.close();+ }+ }+ assertEquals(0L, countWorkingFiles(repository.toPath()));+ }++ private static void writeTable(final File target, final int records)+ throws IOException {+ try (DataOutputStream output = new DataOutputStream(+ new BufferedOutputStream(Files.newOutputStream(target.toPath())))) {+ for (int record = 0; record < records; record++) {+ output.write(key(record));+ output.write(NaturalOrder.encodeLong(offset(record), Long.BYTES));+ }+ }+ }++ private static long countWorkingFiles(final Path repository) throws IOException {+ try (java.util.stream.Stream<Path> files = Files.walk(repository)) {+ return files.filter(path -> path.getFileName().toString()+ .endsWith(".sstwork")).count();+ }+ }++ private static long countBuilderArtifacts(final Path repository) throws IOException {+ try (java.util.stream.Stream<Path> files = Files.walk(repository)) {+ return files.filter(path -> {+ final String name = path.getFileName().toString();+ return name.startsWith("sst-build-")+ && (name.endsWith(".sstlock") || name.endsWith(".run")+ || name.endsWith(".merge") || name.endsWith(".sst"));+ }).count();+ }+ }++ private static byte[] key(final long record) {+ return NaturalOrder.encodeLong(record, KEY_LENGTH);+ }++ private static long offset(final long record) {+ return record * 10L;+ }++ private static boolean isWindows() {+ return System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win");+ }+}diff --git a/test/java/net/yacy/kelondro/index/SSTableHandleMapTest.java b/test/java/net/yacy/kelondro/index/SSTableHandleMapTest.javanew file mode 100644index 000000000..43e889ff0--- /dev/null+++ b/test/java/net/yacy/kelondro/index/SSTableHandleMapTest.java@@ -0,0 +1,344 @@+/*+ * SSTableHandleMapTest+ * Copyright 2026 by Michael Peter Christen; [email protected], Frankfurt a. M., Germany+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ */++package net.yacy.kelondro.index;++import static org.junit.Assert.assertArrayEquals;+import static org.junit.Assert.assertEquals;+import static org.junit.Assert.assertFalse;+import static org.junit.Assert.assertTrue;++import java.io.BufferedOutputStream;+import java.io.DataOutputStream;+import java.io.File;+import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;++import org.junit.Rule;+import org.junit.Test;+import org.junit.rules.TemporaryFolder;++import net.yacy.cora.order.CloneableIterator;+import net.yacy.cora.order.NaturalOrder;++public class SSTableHandleMapTest {++ @Rule+ public final TemporaryFolder temporaryFolder = new TemporaryFolder();++ @Test+ public void builderMergesBoundedRunsAndRemovesItsWorkingFile() throws Exception {+ final File directory = this.temporaryFolder.newFolder("builder");+ final SSTableHandleMap table;+ try (SSTableHandleMap.Builder builder = new SSTableHandleMap.Builder(+ 4, NaturalOrder.naturalOrder, Long.BYTES,+ directory, "multipass", 1)) {+ /* One buffered record per run forces both fan-in limiting and a second merge pass. */+ for (int index = 129; index >= 0; index--) {+ builder.consume(key(index), index);+ }+ builder.consume(key(50), 999L);+ table = builder.finish();+ }++ try {+ assertEquals(130, table.size());+ assertEquals(0L, table.get(key(0)));+ assertEquals(999L, table.get(key(50)));+ assertEquals(129L, table.get(key(129)));++ final CloneableIterator<byte[]> keys = table.keys(true, null);+ int expected = 0;+ while (keys.hasNext()) assertArrayEquals(key(expected++), keys.next());+ assertEquals(130, expected);++ assertEquals(1, directory.list().length);+ } finally {+ table.close();+ }+ assertEquals(0, directory.list().length);+ }++ @Test+ public void workingCopyPromotesLazilyAndHonorsOwnershipLocks() throws Exception {+ final File directory = this.temporaryFolder.newFolder("working-copy");+ final File fingerprint = new File(directory, "heap.fingerprint.idx");+ writeFingerprint(fingerprint);+ final byte[] publishedSnapshot = Files.readAllBytes(fingerprint.toPath());++ /* Simulate a working file left by a crashed process: it has no live lock. */+ final String prefix = "sst-work-" + Integer.toHexString(+ fingerprint.getAbsolutePath().hashCode()) + "-";+ final Path crashedRepository = Files.createDirectories(+ directory.toPath().resolve("repository/old-generation"));+ Files.write(crashedRepository.resolve(prefix + "stale.sstwork"), new byte[] {1});++ final SSTableHandleMap.RecoveryReport initialRecovery =+ SSTableHandleMap.recoverWorkingCopies(directory, crashedRepository.toFile());+ assertEquals(1L, initialRecovery.candidates());+ assertEquals(1L, initialRecovery.deleted());+ assertEquals(0L, initialRecovery.inUse());+ assertEquals(0L, initialRecovery.failures());++ final SSTableHandleMap first = SSTableHandleMap.openWorkingCopy(+ 4, NaturalOrder.naturalOrder, Long.BYTES, fingerprint);+ SSTableHandleMap second = null;+ try {+ /* Reads stay directly on the published read-only mapping. */+ assertEquals(0, countWorkingFiles(directory.toPath()));+ assertEquals(20L, first.remove(key(2)));+ assertEquals(1, countWorkingFiles(directory.toPath()));++ /* The tombstone belongs to the private generation, never to the snapshot. */+ assertArrayEquals(publishedSnapshot, Files.readAllBytes(fingerprint.toPath()));++ second = SSTableHandleMap.openWorkingCopy(+ 4, NaturalOrder.naturalOrder, Long.BYTES, fingerprint);+ assertEquals(1, countWorkingFiles(directory.toPath()));+ assertEquals(20L, second.get(key(2)));+ assertEquals(30L, second.remove(key(3)));+ assertEquals(2, countWorkingFiles(directory.toPath()));++ /* A global pass may run defensively without touching live generations. */+ final SSTableHandleMap.RecoveryReport liveRecovery =+ SSTableHandleMap.recoverWorkingCopies(directory);+ assertEquals(2L, liveRecovery.candidates());+ assertEquals(0L, liveRecovery.deleted());+ assertEquals(2L, liveRecovery.inUse());+ assertEquals(0L, liveRecovery.failures());+ } finally {+ if (second != null) second.close();+ first.close();+ }+ assertEquals(0, countWorkingFiles(directory.toPath()));+ assertArrayEquals(publishedSnapshot, Files.readAllBytes(fingerprint.toPath()));+ }++ @Test+ public void recoveryCoversBuilderArtifactsAndPreservesUnrelatedFiles() throws Exception {+ final File directory = this.temporaryFolder.newFolder("builder-recovery");+ final String session = "sst-build-deadbeef-0123456789abcdef0123456789abcdef";+ final String[] artifacts = {+ session + ".sstlock",+ session + "-1.run",+ session + "-2.merge",+ session + "-3.sst",+ "sst-deadbeef-legacy.run",+ "sst-deadbeef-legacy.merge",+ "sst-deadbeef-legacy.sst"+ };+ for (final String artifact : artifacts) {+ Files.write(directory.toPath().resolve(artifact), new byte[] {1});+ }++ final Path unrelatedRun = Files.write(+ directory.toPath().resolve("application.run"), new byte[] {2});+ final Path unrelatedSSTable = Files.write(+ directory.toPath().resolve("sst-user-data.sst"), new byte[] {3});++ final SSTableHandleMap.RecoveryReport recovery =+ SSTableHandleMap.recoverWorkingCopies(directory);+ assertEquals(artifacts.length, recovery.candidates());+ assertEquals(artifacts.length, recovery.deleted());+ assertEquals(0L, recovery.inUse());+ assertEquals(0L, recovery.failures());+ for (final String artifact : artifacts) {+ assertFalse(Files.exists(directory.toPath().resolve(artifact)));+ }+ assertTrue(Files.isRegularFile(unrelatedRun));+ assertTrue(Files.isRegularFile(unrelatedSSTable));+ }++ @Test+ public void recoveryDoesNotDeleteActiveBuilderOrFinishedTable() throws Exception {+ final File directory = this.temporaryFolder.newFolder("active-builder-recovery");+ final SSTableHandleMap.Builder builder = new SSTableHandleMap.Builder(+ 4, NaturalOrder.naturalOrder, Long.BYTES,+ directory, "active", 1);+ SSTableHandleMap table = null;+ try {+ builder.consume(key(1), 10L);++ final SSTableHandleMap.RecoveryReport whileBuilding =+ SSTableHandleMap.recoverWorkingCopies(directory);+ assertEquals(2L, whileBuilding.candidates());+ assertEquals(0L, whileBuilding.deleted());+ assertEquals(2L, whileBuilding.inUse());+ assertEquals(0L, whileBuilding.failures());++ table = builder.finish();+ builder.close();+ assertEquals(10L, table.get(key(1)));++ final SSTableHandleMap.RecoveryReport whileOpen =+ SSTableHandleMap.recoverWorkingCopies(directory);+ assertEquals(1L, whileOpen.candidates());+ assertEquals(0L, whileOpen.deleted());+ assertEquals(1L, whileOpen.inUse());+ assertEquals(0L, whileOpen.failures());+ assertEquals(10L, table.get(key(1)));+ } finally {+ try {+ builder.close();+ } finally {+ if (table != null) table.close();+ }+ }+ assertEquals(0, directory.list().length);+ }++ @Test+ public void workingCopyDecompressesGzipFingerprint() throws Exception {+ final File directory = this.temporaryFolder.newFolder("working-copy-gzip");+ final File fingerprint = new File(directory, "heap.fingerprint.idx.gz");+ writeFingerprint(fingerprint);++ final SSTableHandleMap table = SSTableHandleMap.openWorkingCopy(+ 4, NaturalOrder.naturalOrder, Long.BYTES, fingerprint);+ try {+ assertEquals(3, table.size());+ assertEquals(30L, table.get(key(3)));+ assertEquals(1, countWorkingFiles(directory.toPath()));+ final SSTableHandleMap.RecoveryReport recovery =+ SSTableHandleMap.recoverWorkingCopies(directory);+ assertEquals(1L, recovery.candidates());+ assertEquals(0L, recovery.deleted());+ assertEquals(1L, recovery.inUse());+ assertEquals(0L, recovery.failures());+ } finally {+ table.close();+ }+ assertEquals(0, countWorkingFiles(directory.toPath()));+ }++ @Test+ public void clearAndRemoveOnePromoteOnlyWhenTheyChangeTheTable() throws Exception {+ final File directory = this.temporaryFolder.newFolder("working-copy-mutations");+ final File fingerprint = new File(directory, "heap.fingerprint.idx");+ writeFingerprint(fingerprint);+ final byte[] publishedSnapshot = Files.readAllBytes(fingerprint.toPath());++ final SSTableHandleMap removeOne = SSTableHandleMap.openWorkingCopy(+ 4, NaturalOrder.naturalOrder, Long.BYTES, fingerprint);+ try {+ assertEquals(0, countWorkingFiles(directory.toPath()));+ assertEquals(30L, removeOne.removeone());+ assertEquals(1, countWorkingFiles(directory.toPath()));+ } finally {+ removeOne.close();+ }++ final SSTableHandleMap clear = SSTableHandleMap.openWorkingCopy(+ 4, NaturalOrder.naturalOrder, Long.BYTES, fingerprint);+ try {+ assertEquals(0, countWorkingFiles(directory.toPath()));+ clear.clear();+ assertEquals(0, clear.size());+ assertEquals(1, countWorkingFiles(directory.toPath()));+ clear.clear();+ assertEquals(1, countWorkingFiles(directory.toPath()));+ } finally {+ clear.close();+ }++ assertEquals(0, countWorkingFiles(directory.toPath()));+ assertArrayEquals(publishedSnapshot, Files.readAllBytes(fingerprint.toPath()));+ }++ @Test+ public void rejectsTruncatedUnsortedAndDuplicateTables() throws Exception {+ final File directory = this.temporaryFolder.newFolder("invalid-tables");++ final File truncated = new File(directory, "truncated.idx");+ Files.write(truncated.toPath(), new byte[11]);+ assertInvalidTable(truncated, "not a multiple");++ final File unsorted = new File(directory, "unsorted.idx");+ writeRawTable(unsorted, new int[] {1, 3, 2, 4}, null);+ assertInvalidTable(unsorted, "out of order");++ final File duplicate = new File(directory, "duplicate.idx");+ writeRawTable(duplicate, new int[] {1, 2, 2, 3}, null);+ assertInvalidTable(duplicate, "duplicate key");+ }++ @Test+ public void rejectsUnknownNegativeOffsetsButAcceptsTombstones() throws Exception {+ final File directory = this.temporaryFolder.newFolder("invalid-offsets");+ final File invalid = new File(directory, "negative.idx");+ writeRawTable(invalid, new int[] {1, 2, 3}, new long[] {10L, -2L, 30L});+ assertInvalidTable(invalid, "negative SSTable offset -2");++ final File tombstoned = new File(directory, "tombstoned.idx");+ writeRawTable(tombstoned, new int[] {1, 2, 3}, new long[] {10L, -1L, 30L});+ final SSTableHandleMap table = new SSTableHandleMap(+ 4, NaturalOrder.naturalOrder, Long.BYTES, tombstoned);+ try {+ assertEquals(2, table.size());+ assertEquals(-1L, table.get(key(2)));+ assertEquals(30L, table.get(key(3)));+ } finally {+ table.close();+ }+ }++ private static void writeFingerprint(final File target) throws Exception {+ final RowHandleMap map = new RowHandleMap(+ 4, NaturalOrder.naturalOrder, Long.BYTES, 3, target.getName());+ try {+ map.putUnique(key(1), 10L);+ map.putUnique(key(2), 20L);+ map.putUnique(key(3), 30L);+ map.dump(target);+ } finally {+ map.close();+ }+ }++ private static void writeRawTable(final File target, final int[] keys,+ final long[] offsets) throws IOException {+ try (DataOutputStream output = new DataOutputStream(+ new BufferedOutputStream(Files.newOutputStream(target.toPath())))) {+ for (int index = 0; index < keys.length; index++) {+ output.write(key(keys[index]));+ output.write(NaturalOrder.encodeLong(+ offsets == null ? keys[index] * 10L : offsets[index],+ Long.BYTES));+ }+ }+ }++ private static void assertInvalidTable(final File tableFile,+ final String expectedMessage) throws Exception {+ SSTableHandleMap table = null;+ try {+ table = new SSTableHandleMap(+ 4, NaturalOrder.naturalOrder, Long.BYTES, tableFile);+ throw new AssertionError("Invalid SSTable was accepted: " + tableFile);+ } catch (final IOException expected) {+ assertTrue(expected.getMessage(),+ expected.getMessage().contains(expectedMessage));+ } finally {+ if (table != null) table.close();+ }+ }++ private static long countWorkingFiles(final Path directory) throws Exception {+ try (java.util.stream.Stream<Path> files = Files.walk(directory)) {+ return files.filter(path -> path.getFileName().toString().endsWith(".sstwork")).count();+ }+ }++ private static byte[] key(final int value) {+ return String.format("%04d", value).getBytes(StandardCharsets.US_ASCII);+ }+}