Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
added Transformer from the llama3.java project This is the one-file "toy" transformer from https://github.com/mukel/llama3.java in an enhanced form:
Michael Peter Christenb0a94aba8f33
diff --git a/source/net/yacy/ai/llama3/ChatFormat.java b/source/net/yacy/ai/llama3/ChatFormat.javanew file mode 100644index 000000000..871b83d05--- /dev/null+++ b/source/net/yacy/ai/llama3/ChatFormat.java@@ -0,0 +1,130 @@+/**+ * ChatFormat.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3;++import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.Set;++import net.yacy.ai.llama3.Model.Tokenizer;++/**+ * Utility tailored for Llama 3 instruct prompt format.+ */+public class ChatFormat {++ final Tokenizer tokenizer;+ final int beginOfText;+ final int endHeader;+ final int startHeader;+ final int endOfTurn;+ final int endOfText;+ final int endOfMessage;+ final Set<Integer> stopTokens;++ public ChatFormat(Tokenizer tokenizer) {+ this.tokenizer = tokenizer;+ Map<String, Integer> specialTokens = this.tokenizer.getSpecialTokens();+ this.beginOfText = specialTokens.get("<|begin_of_text|>");+ this.startHeader = specialTokens.get("<|start_header_id|>");+ this.endHeader = specialTokens.get("<|end_header_id|>");+ this.endOfTurn = specialTokens.get("<|eot_id|>");+ this.endOfText = specialTokens.get("<|end_of_text|>");+ this.endOfMessage = specialTokens.getOrDefault("<|eom_id|>", -1); // only in 3.1+ this.stopTokens = Set.of(endOfText, endOfTurn);+ }++ public Tokenizer getTokenizer() {+ return tokenizer;+ }++ public Set<Integer> getStopTokens() {+ return stopTokens;+ }++ public List<Integer> encodeHeader(ChatFormat.Message message) {+ List<Integer> tokens = new ArrayList<>();+ tokens.add(startHeader);+ tokens.addAll(this.tokenizer.encodeAsList(message.role().name()));+ tokens.add(endHeader);+ tokens.addAll(this.tokenizer.encodeAsList("\n"));+ return tokens;+ }++ public List<Integer> encodeMessage(ChatFormat.Message message) {+ List<Integer> tokens = this.encodeHeader(message);+ tokens.addAll(this.tokenizer.encodeAsList(message.content().strip()));+ tokens.add(endOfTurn);+ return tokens;+ }++ public List<Integer> encodeDialogPrompt(boolean appendAssistantTurn, List<ChatFormat.Message> dialog) {+ List<Integer> tokens = new ArrayList<>();+ tokens.add(beginOfText);+ for (ChatFormat.Message message : dialog) {+ tokens.addAll(this.encodeMessage(message));+ }+ if (appendAssistantTurn) {+ // Add the start of an assistant message for the model to complete.+ tokens.addAll(this.encodeHeader(new ChatFormat.Message(ChatFormat.Role.ASSISTANT, "")));+ }+ return tokens;+ }++ public static final class Message {+ private final ChatFormat.Role role;+ private final String content;++ public Message(ChatFormat.Role role, String content) {+ this.role = role;+ this.content = content;+ }++ public ChatFormat.Role role() {+ return role;+ }++ public String content() {+ return content;+ }++ }++ public static final class Role {+ private final String name;++ public static final Role SYSTEM = new Role("system");+ public static final Role USER = new Role("user");+ public static final Role ASSISTANT = new Role("assistant");++ public Role(String name) {+ this.name = name;+ }++ public String name() {+ return name;+ }++ }+}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Context.java b/source/net/yacy/ai/llama3/Context.javanew file mode 100644index 000000000..b4cfbe1b2--- /dev/null+++ b/source/net/yacy/ai/llama3/Context.java@@ -0,0 +1,61 @@+/**+ * Context+ * Copyright 2025 by Michael Peter Christen+ * First released 25.05.2025 at https://yacy.net+ *+ ** This class was not part of the original llama3 implementation,+ ** but added later by the author to support different architectures.+ ** It therefore does not inherit the llama3 copyright.+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.llama3;+++public final class Context {++ public final String prompt;+ public final String systemPrompt;+ public final float temp;+ public final float topp;+ public final long seed;+ public final int maxTokens;++ public static final int DEFAULT_MAX_TOKENS = 512;++ /**+ * Create a context.+ *+ * @param prompt the prompt to use for the model+ * @param systemPrompt a system prompt to use for the model+ * @param temp temperature for sampling, must be >= 0; 0 means greedy sampling (deterministic)+ * @param topp if 0 <= topp <= 1, use top-p (nucleus) sampling; otherwise, use categorical sampling+ * @param seed random seed for sampling; use 0 for a random seed+ * @param maxTokens maximum number of tokens to generate; use 0 for no limit (default is 512)+ */+ public Context(String prompt, String systemPrompt,+ float temp, float topp, long seed, int maxTokens) {+ assert(0 <= temp): "temperature must be positive";+ assert(0 <= topp && topp <= 1): "top-p must be between 0 and 1";++ this.prompt = prompt;+ this.systemPrompt = systemPrompt;+ this.temp = temp;+ this.topp = topp;+ this.seed = seed;+ this.maxTokens = maxTokens;+ }+}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Llama.java b/source/net/yacy/ai/llama3/Llama.javanew file mode 100644index 000000000..e6d04bffd--- /dev/null+++ b/source/net/yacy/ai/llama3/Llama.java@@ -0,0 +1,427 @@+/**+ * Llama.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3;++import java.nio.FloatBuffer;+import java.util.*;+import java.util.function.IntConsumer;+import java.util.stream.IntStream;+import java.util.stream.Stream;++import net.yacy.ai.llama3.Model.Arch;+import net.yacy.ai.llama3.Model.Tokenizer;+import net.yacy.ai.llama3.Tensor.ArrayFloatTensor;+import net.yacy.ai.llama3.Tensor.FloatTensor;++public final class Llama {++ private final Configuration configuration;+ private final Tokenizer tokenizer;+ private final Weights weights;++ public Llama(Configuration configuration, Tokenizer tokenizer, Weights weights) {+ this.configuration = configuration;+ this.tokenizer = tokenizer;+ this.weights = weights;+ }++ public Configuration configuration() {+ return configuration;+ }++ public Tokenizer tokenizer() {+ return tokenizer;+ }++ public Weights weights() {+ return weights;+ }++ public State createNewState(int batchsize) {+ State state = new State(configuration(), batchsize);+ state.latestToken = tokenizer().getSpecialTokens().get("<|begin_of_text|>");+ return state;+ }++ public static final class Configuration {+ public final Arch arch; // architecture+ public final int dim; // transformer dimension+ public final int hiddenDim; // for ffn layers+ public final int numberOfLayers; // number of layers+ public final int numberOfHeads; // number of query heads+ public final int numberOfKeyValueHeads; // number of key/value heads (can be < query heads because of multiquery)+ public final int vocabularySize; // vocabulary size, usually 256 (byte-level)+ public final int contextLength; // max sequence length+ public final boolean sharedWeights;+ public final float rmsNormEps;+ public final float ropeTheta;+ public final int headSize;++ public Configuration(Arch arch, int dim, int hiddenDim, int numberOfLayers, int numberOfHeads, int numberOfKeyValueHeads,+ int vocabularySize, int contextLength, boolean sharedWeights, float rmsNormEps, float ropeTheta) {+ this.arch = arch;+ this.dim = dim;+ this.hiddenDim = hiddenDim;+ this.numberOfLayers = numberOfLayers;+ this.numberOfHeads = numberOfHeads;+ this.numberOfKeyValueHeads = numberOfKeyValueHeads;+ this.vocabularySize = vocabularySize;+ this.contextLength = contextLength;+ this.sharedWeights = sharedWeights;+ this.rmsNormEps = rmsNormEps;+ this.ropeTheta = ropeTheta;+ this.headSize = dim / numberOfHeads;+ }++ public Configuration withContextLength(int newContextLength) {+ if (newContextLength < 0) {+ return this; // no change+ }+ return new Configuration(this.arch, this.dim, this.hiddenDim, this.numberOfLayers, this.numberOfHeads,+ this.numberOfKeyValueHeads, this.vocabularySize, newContextLength,+ this.sharedWeights, this.rmsNormEps, this.ropeTheta);+ }++ }++ public static final class Weights {+ // token embedding table+ public final FloatTensor token_embedding_table; // (vocab_size, dim)+ // weights for rmsnorms+ public final FloatBuffer[] rms_att_weight; // (layer, dim) rmsnorm weights+ // weights for matmuls+ public final FloatTensor[] wq; // (layer, n_heads * head_size)+ public final FloatTensor[] wk; // (layer, n_kv_heads, head_size)+ public final FloatTensor[] wv; // (layer, n_kv_heads * head_size)+ public final FloatTensor[] wo; // (layer, n_heads * head_size, dim)+ public final FloatTensor[] q_bias; // (layer, dim)+ public final FloatTensor[] k_bias; // (layer, kv_dim)+ public final FloatTensor[] v_bias; // (layer, kv_dim)+ public final FloatBuffer[] rms_ffn_weight; // (layer, dim)+ // weights for ffn+ public final FloatTensor[] w1; // (layer, hidden_dim, dim)+ public final FloatTensor[] w2; // (layer, dim, hidden_dim)+ public final FloatTensor[] w3; // (layer, hidden_dim, dim)+ // public final rmsnorm+ public final FloatBuffer rms_final_weight; // (dim,)+ // freq_cis for RoPE relatively positional embeddings+ public final FloatBuffer freq_cis_real; // (seq_len, head_size/2)+ public final FloatBuffer freq_cis_imag; // (seq_len, head_size/2)+ // (optional) classifier weights for the logits, on the last layer+ public final FloatTensor wcls; // (vocab_size, dim)++ public Weights(FloatTensor token_embedding_table, FloatBuffer[] rms_att_weight, FloatTensor[] wq,+ FloatTensor[] wk, FloatTensor[] wv,+ FloatTensor[] q_bias, FloatTensor[] k_bias, FloatTensor[] v_bias,+ FloatTensor[] wo, FloatBuffer[] rms_ffn_weight,+ FloatTensor[] w1, FloatTensor[] w2, FloatTensor[] w3, FloatBuffer rms_final_weight,+ FloatBuffer freq_cis_real, FloatBuffer freq_cis_imag, FloatTensor wcls) {+ this.token_embedding_table = token_embedding_table;+ this.rms_att_weight = rms_att_weight;+ this.wq = wq;+ this.wk = wk;+ this.wv = wv;+ this.q_bias = q_bias;+ this.k_bias = k_bias;+ this.v_bias = v_bias;+ this.wo = wo;+ this.rms_ffn_weight = rms_ffn_weight;+ this.w1 = w1;+ this.w2 = w2;+ this.w3 = w3;+ this.rms_final_weight = rms_final_weight;+ this.freq_cis_real = freq_cis_real;+ this.freq_cis_imag = freq_cis_imag;+ this.wcls = wcls;+ }++ }++ public static final class State {++ // current wave of activations+ public final int batchsize;+ public final FloatTensor[] x; // activation at current time stamp (dim,)+ public final FloatTensor[] xb; // same, but inside a residual branch (dim,)+ public final FloatTensor[] xb2; // an additional buffer just for convenience (dim,)+ public final FloatTensor[] hb; // buffer for hidden dimension in the ffn (hidden_dim,)+ public final FloatTensor[] hb2; // buffer for hidden dimension in the ffn (hidden_dim,)+ public final FloatTensor[] q; // query (dim,)+ public final FloatTensor[] k; // key (dim,)+ public final FloatTensor[] v; // value (dim,)+ public final FloatTensor[] att; // buffer for scores/attention values (n_heads, seq_len)+ public final FloatTensor logits; // output logits++ // kv cache+ public final FloatTensor[] keyCache; // (n_layer, seq_len, kv_dim)+ public final FloatTensor[] valueCache; // (n_layer, seq_len, kv_dim)++ /** last index in previous block */+ int idxPrevBlock;++ public int latestToken;++ State(Configuration config, int batchsize) {+ this.batchsize = batchsize;+ this.x = allocate(batchsize, config.dim);+ this.xb = allocate(batchsize, config.dim);+ this.xb2 = allocate(batchsize, config.dim);+ this.hb = allocate(batchsize, config.hiddenDim);+ this.hb2 = allocate(batchsize, config.hiddenDim);+ this.q = allocate(batchsize, config.dim);+ this.k = allocate(batchsize, config.dim);+ this.v = allocate(batchsize, config.dim);+ this.att = allocate(batchsize, config.numberOfHeads, config.contextLength);+ idxPrevBlock = -1;++ this.logits = ArrayFloatTensor.allocate(config.vocabularySize);+ int kvDim = (config.dim * config.numberOfKeyValueHeads) / config.numberOfHeads;+ this.keyCache = Stream.generate(() -> ArrayFloatTensor.allocate(config.contextLength, kvDim)).limit(config.numberOfLayers).toArray(FloatTensor[]::new);+ this.valueCache = Stream.generate(() -> ArrayFloatTensor.allocate(config.contextLength, kvDim)).limit(config.numberOfLayers).toArray(FloatTensor[]::new);+ }++ private static FloatTensor[] allocate(int numTokens, int... dims) {+ return IntStream.range(0, numTokens)+ .mapToObj(i -> ArrayFloatTensor.allocate(dims))+ .toArray(FloatTensor[]::new);+ }++ }++ static void rmsnorm(FloatTensor out, FloatTensor x, FloatBuffer weight, int size, float rmsNormEps) {+ // calculate sum of squares+ float ss = x.reduce(0, size, 0f, (acc, xi) -> acc + xi * xi);+ ss /= size;+ ss += rmsNormEps;+ ss = (float) (1.0 / Math.sqrt(ss));+ // normalize and scale+ for (int i = 0; i < size; ++i) {+ out.setFloat(i, weight.get(i) * (ss * x.getFloat(i)));+ }+ }++ static FloatTensor forward(Llama model, State state, int[] tokens, int position, boolean computeLogits) {+ // a few convenience variables+ Configuration config = model.configuration();+ Weights weights = model.weights();+ int dim = config.dim;+ int headSize = config.headSize;+ int kvDim = (config.dim * config.numberOfKeyValueHeads) / config.numberOfHeads;+ int kvMul = config.numberOfHeads / config.numberOfKeyValueHeads; // integer multiplier of the kv sharing in multiquery+ float sqrtHeadSize = (float) Math.sqrt(headSize);+ final int nTokens = tokens.length;++ // copy the token embedding into x+ FloatTensor.parallelFor(0, nTokens, t ->+ weights.token_embedding_table.copyTo(tokens[t] * dim, state.x[t], 0, dim)+ );++ // forward all the layers+ for (int l = 0; l < config.numberOfLayers; l++) {++ // attention rmsnorm+ final int curLayer = l;+ FloatTensor.parallelFor(0, nTokens, t ->+ rmsnorm(state.xb[t], state.x[t], weights.rms_att_weight[curLayer], dim, config.rmsNormEps)+ );++ // qkv matmuls for this position+ weights.wq[l].matmul(nTokens, state.xb, state.q, dim, dim);+ weights.wk[l].matmul(nTokens, state.xb, state.k, kvDim, dim);+ weights.wv[l].matmul(nTokens, state.xb, state.v, kvDim, dim);++ // RoPE relative positional encoding: complex-valued rotate q and k in each head+ FloatTensor.parallelFor(0, nTokens, t -> {+ for (int i = 0; i < dim; i += 2) {+ int head_dim = i % headSize;+ float fcr = weights.freq_cis_real.get((position + t) * (headSize / 2) + (head_dim / 2));+ float fci = weights.freq_cis_imag.get((position + t) * (headSize / 2) + (head_dim / 2));+ int rotn = i < kvDim ? 2 : 1; // how many vectors? 2 = q & k, 1 = q only+ for (int vi = 0; vi < rotn; vi++) {+ FloatTensor vec = vi == 0 ? state.q[t] : state.k[t]; // the vector to rotate (query or key)+ float v0 = vec.getFloat(i);+ float v1 = vec.getFloat(i + 1);+ vec.setFloat(i, v0 * fcr - v1 * fci);+ vec.setFloat(i + 1, v0 * fci + v1 * fcr);+ }+ }+ });++ // save key,value at this time step (position) to our kv cache+ FloatTensor.parallelFor(0, nTokens, t -> {+ state.k[t].copyTo(0, state.keyCache[curLayer], (position + t) * kvDim, kvDim);+ state.v[t].copyTo(0, state.valueCache[curLayer], (position + t) * kvDim, kvDim);+ });++ // If the logits are not required, the attention and FFN of the last layer can be skipped entirely.+ if (!computeLogits && curLayer == config.numberOfLayers - 1) {+ state.idxPrevBlock = nTokens - 1;+ return null;+ }++ // multihead attention. iterate over all heads+ FloatTensor.parallelForLong(0, (long) nTokens * (long) config.numberOfHeads, ht -> {+ int token = (int) (ht / config.numberOfHeads);+ int h = (int) (ht % config.numberOfHeads);+ int qOffset = h * headSize;+ int attOffset = h * config.contextLength;++ for (int t = 0; t <= position + token; t++) {+ int keyCacheOffset = t * kvDim + (h / kvMul) * headSize;+ float score = state.q[token].dot(qOffset, state.keyCache[curLayer], keyCacheOffset, headSize);+ score /= sqrtHeadSize;+ state.att[token].setFloat(attOffset + t, score);+ }++ state.att[token].softmaxInPlace(attOffset, position + token + 1);++ int xbOffset = h * headSize;+ state.xb[token].fillInPlace(xbOffset, headSize, 0f);++ for (int t = 0; t <= position + token; t++) {+ int vOffset = t * kvDim + (h / kvMul) * headSize;+ float a = state.att[token].getFloat(attOffset + t);+ state.xb[token].saxpyInPlace(xbOffset, state.valueCache[curLayer], vOffset, headSize, a);+ }+ });++ // final matmul to get the output of the attention+ weights.wo[l].matmul(nTokens, state.xb, state.xb2, dim, dim);++ // residual connection back into x+ FloatTensor.parallelFor(0, nTokens, t -> {+ state.x[t].addInPlace(state.xb2[t]);+ });++ // ffn rmsnorm+ FloatTensor.parallelFor(0, nTokens, t -> {+ rmsnorm(state.xb[t], state.x[t], weights.rms_ffn_weight[curLayer], dim, config.rmsNormEps);+ });++ // Now for FFN in PyTorch we have: self.w2(F.silu(self.w1(x)) * self.w3(x))+ weights.w1[l].matmul(nTokens, state.xb, state.hb, config.hiddenDim, dim);+ weights.w3[l].matmul(nTokens, state.xb, state.hb2, config.hiddenDim, dim);++ // SwiGLU non-linearity+ FloatTensor.parallelFor(0, nTokens, t -> {+ state.hb[t].mapInPlace(value -> value / (float) (1.0 + Math.exp(-value)));+ });++ // elementwise multiply with w3(x)+ FloatTensor.parallelFor(0, nTokens, t -> {+ state.hb[t].multiplyInPlace(state.hb2[t]);+ });++ // final matmul to get the output of the ffn+ weights.w2[l].matmul(nTokens, state.hb, state.xb, dim, config.hiddenDim);++ // residual connection+ FloatTensor.parallelFor(0, nTokens, t -> {+ state.x[t].addInPlace(state.xb[t]);+ });+ }++ // final rmsnorm+ FloatTensor.parallelFor(0, nTokens, t -> {+ rmsnorm(state.x[t], state.x[t], weights.rms_final_weight, dim, config.rmsNormEps);+ });++ // classifier into logits+ weights.wcls.matmul(state.x[nTokens - 1], state.logits, config.vocabularySize, dim);+ state.idxPrevBlock = nTokens - 1;++ return state.logits;+ }++ /**+ * LLM generation entry point, ingest prompt tokens and generates new tokens.+ *+ * <p>+ * All prompt tokens are ingested first, then inference starts, until a stop token is found.+ * The returned tokens only include generated/inferred tokens.+ *+ * @param model model to run inference (including weights, configuration, tokenizer ...)+ * @param state state of the model e.g. key/value caches ... this is mutated by this call+ * @param startPosition start prompt ingestion + inference at this position in the context e.g. useful if state was kept across calls (chained generation). 0 implies run with no previous context.+ * @param promptTokens prompt tokens to ingest, all the prompt tokens will be ingested, given there's enough capacity left in the context+ * @param stopTokens set of tokens that abort generation during inference, stop tokens do not affect prompt ingestion+ * @param maxTokens maximum number of tokens (can go up to {@link Configuration#contextLength context length}+ * if this value is negative or greater than {@link Configuration#contextLength context length}+ * @param sampler {@link Sampler strategy} used to select tokens+ * @param echo debugging flag, prints ALL, prompt and inferred tokens, to {@link System#err stderr}+ * @param onTokenGenerated callback, if non-null, it's called every time a token is inferred e.g. it's not called when ingesting prompt tokens+ * @return list of generated/inferred tokens, including the stop token, if any e.g. does not include any token from the prompt+ */+ public static List<Integer> generateTokens(Llama model, State state, int startPosition, List<Integer> promptTokens,+ Set<Integer> stopTokens, int maxTokens, Sampler sampler,+ IntConsumer onTokenGenerated) {+ //long startNanos = System.nanoTime();+ //long startGen = 0;+ if (maxTokens < 0 || model.configuration().contextLength < maxTokens) {+ maxTokens = model.configuration().contextLength;+ }+ List<Integer> generatedTokens = new ArrayList<>(maxTokens);+ int token = state.latestToken; // BOS?+ int nextToken;+ int promptIndex = 0;+ for (int position = startPosition; position < maxTokens; ++position) {+ if (promptIndex < promptTokens.size()) {+ final int nTokens = Math.min(maxTokens - position, Math.min(promptTokens.size() - promptIndex, state.batchsize));+ final int[] tokens = new int[nTokens];+ for (int i = 0; i < nTokens; i++) {+ tokens[i] = promptTokens.get(promptIndex + i);+ //System.out.print(Tokenizer.replaceControlCharacters(model.tokenizer().decode(List.of(tokens[i]))));++ }+ //System.out.format("position=%d, promptIdx=%d, promptSize=%d, tokens=%s%n", position, promptIndex, promptTokens.size(), Arrays.toString(tokens));++ boolean computeLogits = promptIndex + nTokens >= promptTokens.size();+ forward(model, state, tokens, position, computeLogits);+ position += nTokens - 1;+ promptIndex += nTokens;+ if (promptIndex < promptTokens.size()) {+ continue;+ }+ //startGen = System.nanoTime();+ } else {+ forward(model, state, new int[]{token}, position, true);+ }+ nextToken = sampler.sampleToken(state.logits);+ // System.out.print(Tokenizer.replaceControlCharacters(model.tokenizer().decode(List.of(nextToken))));+ generatedTokens.add(nextToken);+ if (onTokenGenerated != null) {+ onTokenGenerated.accept(nextToken);+ }+ if (stopTokens.contains(nextToken)) {+ break;+ }+ state.latestToken = token = nextToken;+ }++ //long elapsedNanos = System.nanoTime() - startNanos;+ //long promptNanos = startGen - startNanos;+ //long genNanos = elapsedNanos - startGen + startNanos;+ // System.out.printf(startPosition + promptIndex + generatedTokens.size(), model.configuration().contextLength, promptTokens.size() / (promptNanos / 1_000_000_000.0), promptTokens.size(), generatedTokens.size() / (genNanos / 1_000_000_000.0), generatedTokens.size());+ return generatedTokens;+ }+}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Llama3.java b/source/net/yacy/ai/llama3/Llama3.javanew file mode 100644index 000000000..8d1ab166f--- /dev/null+++ b/source/net/yacy/ai/llama3/Llama3.java@@ -0,0 +1,155 @@+/**+ * Llama3.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3;++import java.io.IOException;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.List;+import java.util.Scanner;+import java.util.Set;+import java.util.function.IntConsumer;++import net.yacy.ai.llama3.Model.ModelLoader;++public class Llama3 {++ // Batch-size used in prompt evaluation.+ private static final int BATCH_SIZE = Integer.getInteger("llama.BatchSize", 16);+++ static void runInteractive(Llama model, Sampler sampler, Context options) {+ Llama.State state = null;+ List<Integer> conversationTokens = new ArrayList<>();+ ChatFormat chatFormat = new ChatFormat(model.tokenizer());+ conversationTokens.add(chatFormat.beginOfText);+ if (options.systemPrompt != null) {+ conversationTokens.addAll(chatFormat.encodeMessage(new ChatFormat.Message(ChatFormat.Role.SYSTEM, options.systemPrompt)));+ }+ int startPosition = 0;+ @SuppressWarnings("resource")+ Scanner in = new Scanner(System.in);+ while (true) {+ System.out.print("> ");+ System.out.flush();+ String userText = in.nextLine();+ if (state == null) {+ state = model.createNewState(BATCH_SIZE);+ }+ conversationTokens.addAll(chatFormat.encodeMessage(new ChatFormat.Message(ChatFormat.Role.USER, userText)));+ conversationTokens.addAll(chatFormat.encodeHeader(new ChatFormat.Message(ChatFormat.Role.ASSISTANT, "")));+ Set<Integer> stopTokens = chatFormat.getStopTokens();+ List<Integer> responseTokens = Llama.generateTokens(model, state, startPosition, conversationTokens.subList(startPosition, conversationTokens.size()), stopTokens, options.maxTokens, sampler, token -> {+ if (!model.tokenizer().isSpecialToken(token)) {+ System.out.print(model.tokenizer().decode(List.of(token)));+ }+ });+ // Include stop token in the prompt history, but not in the response displayed to the user.+ conversationTokens.addAll(responseTokens);+ startPosition = conversationTokens.size();+ Integer stopToken = null;+ if (!responseTokens.isEmpty() && stopTokens.contains(responseTokens.get(responseTokens.size()-1))) {+ stopToken = responseTokens.get(responseTokens.size()-1);+ responseTokens.remove(responseTokens.size()-1);+ }+ //System.out.println(model.tokenizer().decode(responseTokens));+ if (stopToken == null) {+ System.out.println("Ran out of context length...");+ break;+ }+ }+ }++ public static List<Integer> runInstructOnce(Llama model, Sampler sampler, Context options, IntConsumer onTokenGenerated) {+ Llama.State state = model.createNewState(BATCH_SIZE);+ ChatFormat chatFormat = new ChatFormat(model.tokenizer());++ List<Integer> promptTokens = new ArrayList<>();+ promptTokens.add(chatFormat.beginOfText);+ if (options.systemPrompt != null) {+ promptTokens.addAll(chatFormat.encodeMessage(new ChatFormat.Message(ChatFormat.Role.SYSTEM, options.systemPrompt)));+ }+ //System.out.println("Context after System Prompt: " + toString(model, promptTokens));+ promptTokens.addAll(chatFormat.encodeMessage(new ChatFormat.Message(ChatFormat.Role.USER, options.prompt)));+ //System.out.println("Context after User Prompt: " + toString(model, promptTokens));+ promptTokens.addAll(chatFormat.encodeHeader(new ChatFormat.Message(ChatFormat.Role.ASSISTANT, "")));+ //System.out.println("Context after Assitant Prompt: " + toString(model, promptTokens));++ Set<Integer> stopTokens = chatFormat.getStopTokens();+ List<Integer> responseTokens = Llama.generateTokens(model, state, 0, promptTokens, stopTokens, options.maxTokens, sampler, onTokenGenerated);++ // remove stop token at the end of the response, if present+ if (!responseTokens.isEmpty() && stopTokens.contains(responseTokens.get(responseTokens.size()-1))) {+ responseTokens.remove(responseTokens.size()-1);+ }+ //System.out.println(model.tokenizer().decode(responseTokens));+ return responseTokens;+ }++ public static String toString(Llama model, List<Integer> tokens) {+ return model.tokenizer().decode(tokens);+ }++ public static void main(String[] args) throws IOException {+ // model download paths:+ // https://huggingface.co/mukel/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf+ // https://huggingface.co/mukel/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q8_0.gguf+ // https://huggingface.co/mukel/Llama-3.2-3B-Instruct-GGUF/resolve/main/Llama-3.2-3B-Instruct-Q4_0.gguf++ // performance on iMac x86:+ // semeru 11 : 1.40 T/s+ // semeru 21 : 1.13 T/s+ // GraalVM 21: 2.27 T/s+ // openjdk 21: 3.02 T/s; 3.2 with VarHandle++ Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "Llama-3.2-1B-Instruct-Q4_0.gguf"); // 26.7 T/s/M4 orig jdk 21; 24.9 T/s/M4 Temurin 24; 25.8 T/s/M4 jdk 21; 22.2 T/s/M4 GraalVM21; 9.2 T/s/M4 Semeru 11; 9.2 T/s/M1 Ultra jdk 11+ //Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "Llama-3.2-1B-Instruct-Q8_0.gguf"); // 10.7 T/s/M4 orig jdk 21; 21.2 T/s/M4 Temurin 24; 22 T/s/M4 jdk 21; 18 T/s/M4 GraalVM21; 5.8 T/s/M4 Semeru 11; 6.7 T/s/M1 Ultra jdk 11+ //Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "Llama-3.2-3B-Instruct-Q4_0.gguf"); // 9.8 T/s/M4 orig jdk 21; 9.6 T/s/M4 Temurin 24; 9.3 T/s/M4 jdk 21; 8.0 T/s/M4 GraalVM21; 3.2 T/s/M4 Semeru 11; 3.8 T/s/M1 Ultra jdk 11+ //Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "Llama-3.2-3B-Instruct-Q8_0.gguf"); // 7.2 T/s/M4 jdk 24+ //Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "Meta-Llama-3-8B-Instruct-Q4_0.gguf"); // 3.6 T/s/M4 jdk 24;+ //Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "OLMo-2-0425-1B-Instruct-Q4_0.gguf");+ Context options = new Context("Write a Java program which computes the first 42 prime numbers.", "Be a very good programmer.", 0.0f, 0.95f, 0, 1024);+ Llama model = ModelLoader.loadModel(modelPath, 1024, true);+ // get time+ long startTime = System.currentTimeMillis();+ Sampler sampler = Sampler.selectSampler(model.configuration().vocabularySize, options.temp, options.topp, options.seed);+ List<Integer> resultToken = runInstructOnce(model, sampler, options, token -> {+ if (!model.tokenizer().isSpecialToken(token)) {+ System.out.print(model.tokenizer().decode(List.of(token)));+ }+ });+ long endTime = System.currentTimeMillis();+ System.out.println("\nToken: " + resultToken.size() + ", " + ((double) resultToken.size()) * 1000.0d / ((double) (endTime - startTime)) + " Tokens per second");+ }+}+++++++++++diff --git a/source/net/yacy/ai/llama3/Model/Arch.java b/source/net/yacy/ai/llama3/Model/Arch.javanew file mode 100644index 000000000..816980c63--- /dev/null+++ b/source/net/yacy/ai/llama3/Model/Arch.java@@ -0,0 +1,31 @@+/**+ * Arch+ * Copyright 2025 by Michael Peter Christen+ * First released 25.05.2025 at https://yacy.net+ *+ ** This class was not part of the original llama3 implementation,+ ** but added later by the author to support different architectures.+ ** It therefore does not inherit the llama3 copyright.+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.llama3.Model;+++// architecture of the model, see https://github.com/ggml-org/llama.cpp/blob/master/src/llama-arch.cpp+public enum Arch {+ LLM_ARCH_LLAMA, LLM_ARCH_QWEN2, LLM_ARCH_QWEN3, LLM_ARCH_OLMO2+}diff --git a/source/net/yacy/ai/llama3/Model/GGMLTensorEntry.java b/source/net/yacy/ai/llama3/Model/GGMLTensorEntry.javanew file mode 100644index 000000000..3b2f19e7b--- /dev/null+++ b/source/net/yacy/ai/llama3/Model/GGMLTensorEntry.java@@ -0,0 +1,97 @@+/**+ * GGMLTensorEntry.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Model;++import java.io.IOException;+import java.nio.ByteBuffer;+import java.nio.ByteOrder;+import java.nio.FloatBuffer;+import java.util.Arrays;+import java.util.Objects;++import net.yacy.ai.llama3.Tensor.FloatTensor;+import net.yacy.ai.llama3.Tensor.Q4_0FloatTensor;+import net.yacy.ai.llama3.Tensor.Q8_0FloatTensor;++public final class GGMLTensorEntry {++ private final ByteBuffer buffer;+ private final String name;+ private final GGMLType ggmlType;+ private final int[] shape;+++ public GGMLTensorEntry(String name, GGMLType ggmlType, int[] shape, ByteBuffer buffer) throws IOException {+ assert buffer.isDirect() : "Buffer must be a direct ByteBuffer";+ this.buffer = buffer;+ buffer.order(ByteOrder.nativeOrder()); // Set correct byte order+ this.name = Objects.requireNonNull(name);+ this.ggmlType = Objects.requireNonNull(ggmlType);+ this.shape = Arrays.copyOf(Objects.requireNonNull(shape), shape.length);+ }++ public float getFloat(int index) {+ return buffer.getFloat(index * Float.BYTES);+ }++ public String name() {+ return name;+ }++ public GGMLType ggmlType() {+ return ggmlType;+ }++ public int[] shape() {+ return Arrays.copyOf(shape, shape.length);+ }++ public FloatBuffer toFloatBuffer() {+ switch (ggmlType) {+ case F32: return buffer.order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer();+ default: throw new UnsupportedOperationException("Conversion to " + ggmlType);+ }+ }++ public FloatTensor loadQuantized() {+ FloatTensor tensor = null;+ switch (ggmlType) {+ //case F32: return new F32FloatTensor(FloatTensor.numberOfElements(entry.shape()), entry.memorySegment());+ case Q8_0: tensor = new Q8_0FloatTensor(FloatTensor.numberOfElements(this.shape()), this.buffer); break;+ case Q4_0: tensor = new Q4_0FloatTensor(FloatTensor.numberOfElements(this.shape()), this.buffer); break;+ default: throw new UnsupportedOperationException("Quantization format " + ggmlType);+ }+ return tensor;++ // create a new ArrayFloatTensor(final float[] values)+ //float[] values = new float[tensor.size()];+ // copy the values from the tensor to the float array+ //for (int i = 0; i < values.length; i++) {+ /// values[i] = tensor.getFloat(i);+ //}+ //return new DirectBufferFloatTensor(values);+ //return new F16FloatTensor(values);+ //return new BF16FloatTensor(values);+ //return new ArrayFloatTensor(values);+ }+}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Model/GGMLType.java b/source/net/yacy/ai/llama3/Model/GGMLType.javanew file mode 100644index 000000000..d0fe6c427--- /dev/null+++ b/source/net/yacy/ai/llama3/Model/GGMLType.java@@ -0,0 +1,99 @@+/**+ * GGMLType.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Model;++public enum GGMLType {+ F32(Float.BYTES),+ F16(GGMLType.FLOAT16_BYTES),+ Q4_0(GGMLType.FLOAT16_BYTES + 16 * Byte.BYTES, 32),+ Q4_1(2 * GGMLType.FLOAT16_BYTES + 16 * Byte.BYTES, 32),+ UNSUPPORTED_Q4_2(Integer.MAX_VALUE), // support has been removed+ UNSUPPORTED_Q4_3(Integer.MAX_VALUE), // support has been removed+ Q5_0(Integer.MAX_VALUE),+ Q5_1(Integer.MAX_VALUE),+ Q8_0(GGMLType.FLOAT16_BYTES + 32 * Byte.BYTES, 32),+ Q8_1(32 * Byte.BYTES + 2 * Float.BYTES, 32),+ // k-quantizations+ Q2_K(Integer.MAX_VALUE),+ Q3_K(Integer.MAX_VALUE),+ Q4_K(2 * GGMLType.FLOAT16_BYTES + ((GGMLType.QK_K / 16) / 8 * 6) + GGMLType.QK_K / 2, GGMLType.QK_K),+ Q5_K(2 * GGMLType.FLOAT16_BYTES + ((GGMLType.QK_K / 16) / 8 * 6) + GGMLType.QK_K / 8 + GGMLType.QK_K / 2, GGMLType.QK_K),+ Q6_K(GGMLType.QK_K / 2 + GGMLType.QK_K / 4 + GGMLType.QK_K / 16 + GGMLType.FLOAT16_BYTES, GGMLType.QK_K),+ Q8_K(Integer.MAX_VALUE),++ IQ2_XXS(Integer.MAX_VALUE),+ IQ2_XS(Integer.MAX_VALUE),+ IQ3_XXS(Integer.MAX_VALUE),+ IQ1_S(Integer.MAX_VALUE),+ IQ4_NL(Integer.MAX_VALUE),+ IQ3_S(Integer.MAX_VALUE),+ IQ2_S(Integer.MAX_VALUE),+ IQ4_XS(Integer.MAX_VALUE),++ I8(Byte.BYTES),+ I16(Short.BYTES),+ I32(Integer.BYTES),+ I64(Long.BYTES),+ F64(Double.BYTES),+ IQ1_M(Integer.MAX_VALUE),+ BF16(GGMLType.BFLOAT16_BYTES),+ Q4_0_4_4(GGMLType.FLOAT16_BYTES + 16 * Byte.BYTES, 32),+ Q4_0_4_8(GGMLType.FLOAT16_BYTES + 16 * Byte.BYTES, 32),+ Q4_0_8_8(GGMLType.FLOAT16_BYTES + 16 * Byte.BYTES, 32),+ TQ1_0(Integer.MAX_VALUE),+ TQ2_0(Integer.MAX_VALUE);++ public static final int BFLOAT16_BYTES = 2;+ public static final int FLOAT16_BYTES = 2;+ private static final GGMLType[] VALUES = values();+ public final int typeSize;+ public final int blockSize;++ public final static GGMLType fromId(int id) {+ return VALUES[id];+ }++ GGMLType(int typeSize) {+ this(typeSize, 1);+ }++ public long byteSizeFor(int numberOfElements) {+ long t = numberOfElements * (long) this.typeSize;+ assert t % this.blockSize == 0;+ return Math.toIntExact(t / this.blockSize);+ }++ public static final int QK_K = 256; // or 64?++ GGMLType(int typeSize, int blockSize) {+ assert blockSize > 0;+ assert typeSize > 0;+ assert isPowerOf2(blockSize);+ this.typeSize = typeSize;+ this.blockSize = blockSize;+ }++ private final static boolean isPowerOf2(int n) {+ return n > 0 && (n & (n - 1)) == 0;+ }+}diff --git a/source/net/yacy/ai/llama3/Model/GGUF.java b/source/net/yacy/ai/llama3/Model/GGUF.javanew file mode 100644index 000000000..6a4fd7209--- /dev/null+++ b/source/net/yacy/ai/llama3/Model/GGUF.java@@ -0,0 +1,645 @@+/**+ * GGUF.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Model;++import java.io.IOException;+import java.nio.ByteBuffer;+import java.nio.ByteOrder;+import java.nio.MappedByteBuffer;+import java.nio.channels.FileChannel;+import java.nio.charset.StandardCharsets;+import java.nio.file.Path;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Comparator;+import java.util.Objects;+import java.util.stream.Collectors;++import net.yacy.ai.llama3.Tensor.FloatTensor;++/*+ * GGUF File Reader. For specification see https://github.com/ggml-org/ggml/blob/master/docs/gguf.md+ */+public final class GGUF {+ private static final int GGUF_MAGIC = 0x46554747;+ private static final int DEFAULT_ALIGNMENT = 32; // must be a power of 2+ private static final List<Integer> SUPPORTED_GGUF_VERSIONS = List.of(2, 3);+ private int magic;+ private int version;+ private int tensorCount; // uint64_t+ private int alignment;+ private int metadata_kv_count; // uint64_t+ private Map<String, Object> metadata;+ private Map<String, GGUFTensorInfo> tensorInfos;+ private long tensorDataOffset;+ private Map<String, GGMLTensorEntry> tensorEntries;++ public Map<String, GGUFTensorInfo> getTensorInfos() {+ return tensorInfos;+ }++ public long getTensorDataOffset() {+ return tensorDataOffset;+ }++ public Map<String, Object> getMetadata() {+ return metadata;+ }++ private final ByteBuffer BB_1 = ByteBuffer.allocate(Byte.BYTES).order(ByteOrder.LITTLE_ENDIAN);+ private final ByteBuffer BB_2 = ByteBuffer.allocate(Short.BYTES).order(ByteOrder.LITTLE_ENDIAN);+ private final ByteBuffer BB_4 = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN);+ private final ByteBuffer BB_8 = ByteBuffer.allocate(Long.BYTES).order(ByteOrder.LITTLE_ENDIAN);++ public Map<String, GGMLTensorEntry> getTensorEntries() {+ return tensorEntries;+ }++ public static GGUF loadModel(Path modelPath) throws IOException {+ try (FileChannel fileChannel = FileChannel.open(modelPath)) {+ GGUF gguf = new GGUF();+ gguf.loadModelImpl(fileChannel);+ return gguf;+ }+ }++ enum MetadataValueType {+ // The value is a 8-bit unsigned integer.+ UINT8(1),+ // The value is a 8-bit signed integer.+ INT8(1),+ // The value is a 16-bit unsigned little-endian integer.+ UINT16(2),+ // The value is a 16-bit signed little-endian integer.+ INT16(2),+ // The value is a 32-bit unsigned little-endian integer.+ UINT32(4),+ // The value is a 32-bit signed little-endian integer.+ INT32(4),+ // The value is a 32-bit IEEE754 floating point number.+ FLOAT32(4),+ // The value is a boolean.+ // 1-byte value where 0 is false and 1 is true.+ // Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy.+ BOOL(1),+ // The value is a UTF-8 non-null-terminated string, with length prepended.+ STRING(-8),+ // The value is an array of other values, with the length and type prepended.+ // Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes.+ ARRAY(-8),+ // The value is a 64-bit unsigned little-endian integer.+ UINT64(8),+ // The value is a 64-bit signed little-endian integer.+ INT64(8),+ // The value is a 64-bit IEEE754 floating point number.+ FLOAT64(8);+ private final int byteSize;++ MetadataValueType(int byteSize) {+ this.byteSize = byteSize;+ }++ private static final MetadataValueType[] VALUES = values();++ public static MetadataValueType fromIndex(int index) {+ return VALUES[index];+ }++ public int byteSize() {+ return byteSize;+ }+ }++ private void loadModelImpl(FileChannel fileChannel) throws IOException {+ // The header of the file.+ readHeader(fileChannel); // gguf_header_t header;+ // Tensor infos, which can be used to locate the tensor data.+ // gguf_tensor_info_t tensor_infos[header.tensor_count];+ this.tensorInfos = new HashMap<>(tensorCount);+ for (int i = 0; i < tensorCount; ++i) {+ GGUF.GGUFTensorInfo ti = readTensorInfo(fileChannel);+ assert !tensorInfos.containsKey(ti.name);+ tensorInfos.put(ti.name, ti);+ }+ // Padding to the nearest multiple of `ALIGNMENT`.+ // uint8_t _padding[ALIGNMENT - (sizeof(header + tensor_infos) % ALIGNMENT)];+ //long _padding = -fileChannel.position() & (ALIGNMENT - 1);+ long _padding = getAlignment() - (fileChannel.position() % getAlignment());+ fileChannel.position(fileChannel.position() + _padding);+ this.tensorDataOffset = fileChannel.position();+ }+ /*+ public static Map<String, GGMLTensorEntry> loadTensors(FileChannel fileChannel, long tensorDataOffset, Map<String, GGUFTensorInfo> tensorInfos) throws IOException {+ // Map the whole remaining file region into memory+ MappedByteBuffer mappedBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, tensorDataOffset, fileChannel.size() - tensorDataOffset);+ mappedBuffer.order(ByteOrder.nativeOrder());++ Map<String, GGMLTensorEntry> tensorEntries = new HashMap<>(tensorInfos.size());++ for (Map.Entry<String, GGUFTensorInfo> entry : tensorInfos.entrySet()) {+ GGUFTensorInfo ti = entry.getValue();+ int numberOfElements = FloatTensor.numberOfElements(ti.dimensions());+ int sizeInBytes = Math.toIntExact(ti.ggmlType().byteSizeFor(numberOfElements));+ int offset = Math.toIntExact(ti.offset()); // assumes offset is within Integer range++ // Create a slice of the mapped buffer for this tensor+ MappedByteBuffer buffer = (MappedByteBuffer) mappedBuffer.duplicate();+ buffer.position(offset);+ buffer.limit(offset + sizeInBytes);++ // copy this into a DirectByteBuffer (tests show that this is not faster than using the MappedByteBuffer directly)+ //ByteBuffer directBuffer = ByteBuffer.allocateDirect(sizeInBytes).order(ByteOrder.nativeOrder());+ //directBuffer.put(buffer);+ //directBuffer.flip();+ //tensorEntries.put(ti.name(), new GGMLTensorEntry(ti.name(), ti.ggmlType(), ti.dimensions(), directBuffer));++ // Create a slice of the mapped buffer for this tensor+ MappedByteBuffer tensorBuffer = (MappedByteBuffer) buffer.slice().order(ByteOrder.nativeOrder());+ tensorEntries.put(ti.name(), new GGMLTensorEntry(ti.name(), ti.ggmlType(), ti.dimensions(), tensorBuffer));+ }++ return tensorEntries;+ }+*/++ static Map<String, GGMLTensorEntry> loadTensors(FileChannel fileChannel, long tensorDataOffset,+ Map<String, GGUFTensorInfo> tensorInfos) throws IOException {++ final long totalDataSize = fileChannel.size() - tensorDataOffset;+ Map<String, GGMLTensorEntry> tensorEntries = new HashMap<>(tensorInfos.size());++ // Fast path: if the entire tensor data fits in one mapped buffer+ if (totalDataSize <= Integer.MAX_VALUE) {+ MappedByteBuffer fullBuffer = (MappedByteBuffer) fileChannel.map(+ FileChannel.MapMode.READ_ONLY,+ tensorDataOffset,+ totalDataSize+ ).order(ByteOrder.nativeOrder());++ for (Map.Entry<String, GGUFTensorInfo> entry : tensorInfos.entrySet()) {+ GGUFTensorInfo ti = entry.getValue();+ long offset = ti.offset();+ int sizeInBytes = Math.toIntExact(ti.ggmlType().byteSizeFor(+ FloatTensor.numberOfElements(ti.dimensions())));++ MappedByteBuffer tensorBuffer = (MappedByteBuffer) fullBuffer.duplicate();+ tensorBuffer.position((int)offset);+ tensorBuffer.limit((int)offset + sizeInBytes);+ tensorBuffer = (MappedByteBuffer) tensorBuffer.slice().order(ByteOrder.nativeOrder());++ tensorEntries.put(ti.name(), new GGMLTensorEntry(+ ti.name(), ti.ggmlType(), ti.dimensions(), tensorBuffer));+ }+ return tensorEntries;+ }++ // Slow path: only for large files > 2GB:+ // Models can be very large, so we cannot map the entire tensor data at once.+ // Instead, we will map segments of the tensor data as needed.++ // First pass: collect all tensor boundaries and sort them+ List<Long> boundaries = new ArrayList<>();+ for (GGUFTensorInfo ti : tensorInfos.values()) {+ long start = ti.offset();+ long end = start + ti.ggmlType().byteSizeFor(FloatTensor.numberOfElements(ti.dimensions()));+ boundaries.add(start);+ boundaries.add(end);+ }+ boundaries = boundaries.stream()+ .distinct()+ .sorted()+ .collect(Collectors.toList());++ // Second pass: create memory mappings for each segment between boundaries+ List<MappedSegment> mappings = new ArrayList<>();+ long currentPos = tensorDataOffset;++ for (long boundary : boundaries) {+ if (boundary <= currentPos) continue;++ long mappingSize = boundary - currentPos;+ // Split into chunks no larger than Integer.MAX_VALUE+ while (mappingSize > 0) {+ long chunkSize = Math.min(mappingSize, Integer.MAX_VALUE);+ MappedByteBuffer buffer = (MappedByteBuffer) fileChannel.map(+ FileChannel.MapMode.READ_ONLY,+ currentPos,+ chunkSize+ ).order(ByteOrder.nativeOrder());+ mappings.add(new MappedSegment(currentPos, buffer));+ currentPos += chunkSize;+ mappingSize -= chunkSize;+ }+ }++ // Handle any remaining data after last boundary+ if (currentPos < fileChannel.size()) {+ long remaining = fileChannel.size() - currentPos;+ while (remaining > 0) {+ long chunkSize = Math.min(remaining, Integer.MAX_VALUE);+ MappedByteBuffer buffer = (MappedByteBuffer) fileChannel.map(+ FileChannel.MapMode.READ_ONLY,+ currentPos,+ chunkSize+ ).order(ByteOrder.nativeOrder());+ mappings.add(new MappedSegment(currentPos, buffer));+ currentPos += chunkSize;+ remaining -= chunkSize;+ }+ }++ // Third pass: create tensor views+ for (Map.Entry<String, GGUFTensorInfo> entry : tensorInfos.entrySet()) {+ GGUFTensorInfo ti = entry.getValue();+ String name = ti.name();+ long tensorOffset = ti.offset() + tensorDataOffset;+ long tensorSize = ti.ggmlType().byteSizeFor(FloatTensor.numberOfElements(ti.dimensions()));+ long tensorEnd = tensorOffset + tensorSize;++ // Find all segments that overlap with this tensor+ List<MappedSegment> overlappingSegments = mappings.stream()+ .filter(seg -> seg.startOffset <= tensorEnd &&+ seg.startOffset + seg.buffer.capacity() > tensorOffset)+ .sorted(Comparator.comparingLong(seg -> seg.startOffset))+ .collect(Collectors.toList());++ if (overlappingSegments.isEmpty()) {+ throw new IOException("Tensor " + name + " not found in any mapped segment");+ }++ if (overlappingSegments.size() == 1) {+ // Simple case - tensor fits entirely in one segment+ MappedSegment segment = overlappingSegments.get(0);+ int bufferOffset = (int)(tensorOffset - segment.startOffset);+ int sizeInBytes = (int)tensorSize;++ MappedByteBuffer tensorBuffer = (MappedByteBuffer) segment.buffer.duplicate();+ tensorBuffer.position(bufferOffset);+ tensorBuffer.limit(bufferOffset + sizeInBytes);+ tensorBuffer = (MappedByteBuffer) tensorBuffer.slice().order(ByteOrder.nativeOrder());++ tensorEntries.put(name, new GGMLTensorEntry(+ name, ti.ggmlType(), ti.dimensions(), tensorBuffer));+ } else {+ // Complex case - tensor spans multiple segments+ ByteBuffer combinedBuffer = ByteBuffer.allocateDirect((int)tensorSize)+ .order(ByteOrder.nativeOrder());++ long remainingBytes = tensorSize;+ long currentTensorPos = tensorOffset;++ for (MappedSegment segment : overlappingSegments) {+ long segmentEnd = segment.startOffset + segment.buffer.capacity();+ long copyStart = Math.max(currentTensorPos, segment.startOffset);+ long copyEnd = Math.min(tensorEnd, segmentEnd);+ int bytesToCopy = (int)(copyEnd - copyStart);++ int srcPos = (int)(copyStart - segment.startOffset);+ segment.buffer.position(srcPos);++ byte[] temp = new byte[bytesToCopy];+ segment.buffer.get(temp);+ combinedBuffer.put(temp);++ remainingBytes -= bytesToCopy;+ currentTensorPos += bytesToCopy;++ if (remainingBytes <= 0) break;+ }++ combinedBuffer.flip();+ tensorEntries.put(name, new GGMLTensorEntry(+ name, ti.ggmlType(), ti.dimensions(), combinedBuffer));+ }+ }++ return tensorEntries;+ }++ // Helper class to track mapped segments+ private static class MappedSegment {+ final long startOffset;+ final MappedByteBuffer buffer;++ MappedSegment(long startOffset, MappedByteBuffer buffer) {+ this.startOffset = startOffset;+ this.buffer = buffer;+ }+ }++ public static final class GGUFTensorInfo {+ private final String name;+ private final int[] dimensions;+ private final GGMLType ggmlType;+ private final long offset;++ public GGUFTensorInfo(String name, int[] dimensions, GGMLType ggmlType, long offset) {+ this.name = name;+ this.dimensions = dimensions != null ? dimensions.clone() : null;+ this.ggmlType = ggmlType;+ this.offset = offset;+ }++ public String name() {+ return name;+ }++ public int[] dimensions() {+ return dimensions != null ? dimensions.clone() : null;+ }++ public GGMLType ggmlType() {+ return ggmlType;+ }++ public long offset() {+ return offset;+ }++ @Override+ public boolean equals(Object o) {+ if (this == o) return true;+ if (o == null || getClass() != o.getClass()) return false;+ GGUFTensorInfo that = (GGUFTensorInfo) o;+ return offset == that.offset &&+ Objects.equals(name, that.name) &&+ Arrays.equals(dimensions, that.dimensions) &&+ Objects.equals(ggmlType, that.ggmlType);+ }++ @Override+ public int hashCode() {+ int result = Objects.hash(name, ggmlType, offset);+ result = 31 * result + Arrays.hashCode(dimensions);+ return result;+ }++ @Override+ public String toString() {+ return "GGUFTensorInfo[" ++ "name=" + name ++ ", dimensions=" + Arrays.toString(dimensions) ++ ", ggmlType=" + ggmlType ++ ", offset=" + offset ++ ']';+ }+ }++ private GGMLType readGGMLType(FileChannel fileChannel) throws IOException {+ int ggmlTypeId = readInt(fileChannel); // ggml_type type;+ return GGMLType.fromId(ggmlTypeId);+ }++ private GGUF.GGUFTensorInfo readTensorInfo(FileChannel fileChannel) throws IOException {+ // The name of the tensor. It is a standard GGUF string, with the caveat that+ // it must be at most 64 bytes long.+ String name = readString(fileChannel); // gguf_string_t name;+ assert name.length() <= 64;+ // The number of dimensions in the tensor.+ // Currently at most 4, but this may change in the future.+ int n_dimensions = readInt(fileChannel); // uint32_t n_dimensions;+ assert n_dimensions <= 4;+ // The dimensions of the tensor.+ int[] dimensions = new int[n_dimensions]; // uint64_t dimensions[n_dimensions];+ for (int i = 0; i < n_dimensions; ++i) {+ dimensions[i] = Math.toIntExact(readLong(fileChannel));+ }+ // The type of the tensor.+ GGMLType ggmlType = readGGMLType(fileChannel); // ggml_type type;+ // The offset of the tensor's data in this file in bytes.+ // This offset is relative to `tensor_data`, not to the start+ // of the file, to make it easier for writers to write the file.+ // Readers should consider exposing this offset relative to the+ // file to make it easier to read the data.+ // Must be a multiple of `ALIGNMENT`.+ long offset = readLong(fileChannel); // uint64_t offset;+ assert offset % getAlignment() == 0;+ return new GGUF.GGUFTensorInfo(name, dimensions, ggmlType, offset);+ }++ private String readString(FileChannel fileChannel) throws IOException {+ // A string in GGUF.+ // The length of the string, in bytes.+ int len = Math.toIntExact(readLong(fileChannel)); // uint64_t len;+ // The string as a UTF-8 non-null-terminated string.+ byte[] bytes = new byte[len]; // char string[len];+ int bytesRead = fileChannel.read(ByteBuffer.wrap(bytes));+ assert len == bytesRead;+ return new String(bytes, StandardCharsets.UTF_8);+ }++ private Pair<String, Object> readKeyValuePair(FileChannel fileChannel) throws IOException {+ // The key of the metadata. It is a standard GGUF string, with the following caveats:+ // - It must be a valid ASCII string.+ // - It must be a hierarchical key, where each segment is `lower_snake_case` and separated by a `.`.+ // - It must be at most 2^16-1/65535 bytes long.+ // Any keys that do not follow these rules are invalid.+ String key = readString(fileChannel); // gguf_string_t key;+ assert key.length() < (1 << 16);+ assert key.codePoints().allMatch(cp -> ('a' <= cp && cp <= 'z') || ('0' <= cp && cp <= '9') || cp == '_' || cp == '.');+ Object value = readMetadataValue(fileChannel);+ return new Pair<>(key, value);+ }++ private Object readMetadataValue(FileChannel fileChannel) throws IOException {+ // The type of the value.+ // Must be one of the `gguf_metadata_value_type` values.+ MetadataValueType value_type = readMetadataValueType(fileChannel); // gguf_metadata_value_type value_type;+ // The value.+ return readMetadataValueOfType(value_type, fileChannel); // gguf_metadata_value_t value;+ }++ // compare this with https://github.com/ggml-org/llama.cpp/blob/master/gguf-py/gguf/gguf_reader.py#L132+ void readHeader(FileChannel fileChannel) throws IOException {+ // Magic number to announce that this is a GGUF file.+ // Must be `GGUF` at the byte level: `0x47` `0x47` `0x55` `0x46`.+ // Your executor might do little-endian byte order, so it might be+ // check for 0x46554747 and letting the endianness cancel out.+ // Consider being *very* explicit about the byte order here.+ this.magic = readInt(fileChannel); // uint32_t magic;+ if (magic != GGUF_MAGIC) {+ throw new IllegalArgumentException("unsupported header.magic " + magic);+ }+ // The version of the format implemented.+ // Must be `3` for version described in this spec.+ //+ // This version should only be increased for structural changes to the format.+ // Changes that do not affect the structure of the file should instead update the metadata+ // to signify the change.+ this.version = readInt(fileChannel); // uint32_t version;+ if (!SUPPORTED_GGUF_VERSIONS.contains(version)) {+ throw new IllegalArgumentException("unsupported header.version " + version);+ }+ // The number of tensors in the file.+ // This is explicit, instead of being included in the metadata, to ensure it is always present+ // for loading the tensors.+ // https://github.com/ggml-org/llama.cpp/blob/master/gguf-py/gguf/gguf_reader.py#L165+ this.tensorCount = Math.toIntExact(readLong(fileChannel)); // uint64_t tensor_count;+ // The number of metadata key-value pairs.+ this.metadata_kv_count = Math.toIntExact(readLong(fileChannel)); // uint64_t metadata_kv_count;+ // The metadata key-value pairs.+ // gguf_metadata_kv_t metadata_kv[metadata_kv_count];+ this.metadata = new HashMap<>(metadata_kv_count);+ for (int i = 0; i < metadata_kv_count; ++i) {+ Pair<String, Object> keyValue = readKeyValuePair(fileChannel);+ assert !metadata.containsKey(keyValue.first());+ metadata.put(keyValue.first(), keyValue.second());+ }+ }++ private Object readArray(FileChannel fileChannel) throws IOException {+ // Any value type is valid, including arrays.+ MetadataValueType value_type = readMetadataValueType(fileChannel); // gguf_metadata_value_type type;+ // Number of elements, not bytes+ int len = Math.toIntExact(readLong(fileChannel)); // uint64_t len;+ // The array of values.+ // gguf_metadata_value_t array[len];+ switch (value_type) {+ case UINT8:+ case INT8: {+ byte[] bytes = new byte[len];+ for (int i = 0; i < len; ++i) {+ bytes[i] = readByte(fileChannel);+ }+ return bytes;+ }+ case UINT16:+ case INT16: {+ short[] shorts = new short[len];+ for (int i = 0; i < len; ++i) {+ shorts[i] = readShort(fileChannel);+ }+ return shorts;+ }+ case UINT32:+ case INT32: {+ int[] ints = new int[len];+ for (int i = 0; i < len; ++i) {+ ints[i] = readInt(fileChannel);+ }+ return ints;+ }+ case FLOAT32: {+ float[] floats = new float[len];+ for (int i = 0; i < len; ++i) {+ floats[i] = readFloat(fileChannel);+ }+ return floats;+ }+ case BOOL: {+ boolean[] booleans = new boolean[len];+ for (int i = 0; i < len; ++i) {+ booleans[i] = readBoolean(fileChannel);+ }+ return booleans;+ }+ case STRING: {+ String[] strings = new String[len];+ for (int i = 0; i < len; ++i) {+ strings[i] = readString(fileChannel);+ }+ return strings;+ }+ case ARRAY: {+ Object[] arrays = new Object[len];+ for (int i = 0; i < len; ++i) {+ arrays[i] = readArray(fileChannel);+ }+ return arrays;+ }+ default: throw new UnsupportedOperationException("read array of " + value_type);+ }+ }++ private Object readMetadataValueOfType(MetadataValueType valueType, FileChannel fileChannel) throws IOException {+ switch (valueType) {+ case UINT8:+ case INT8: return readByte(fileChannel);+ case UINT16:+ case INT16: return readShort(fileChannel);+ case UINT32:+ case INT32: return readInt(fileChannel);+ case FLOAT32: return readFloat(fileChannel);+ case UINT64:+ case INT64: return readLong(fileChannel);+ case FLOAT64: return readDouble(fileChannel);+ case BOOL: return readBoolean(fileChannel);+ case STRING: return readString(fileChannel);+ case ARRAY: return readArray(fileChannel);+ default: throw new AssertionError();+ }+ }++ private byte readByte(FileChannel fileChannel) throws IOException {+ int bytesRead = fileChannel.read(BB_1);+ assert bytesRead == 1;+ return BB_1.clear().get(0);+ }++ private boolean readBoolean(FileChannel fileChannel) throws IOException {+ return readByte(fileChannel) != 0;+ }++ private short readShort(FileChannel fileChannel) throws IOException {+ int bytesRead = fileChannel.read(BB_2);+ assert bytesRead == 2;+ return BB_2.clear().getShort(0);+ }++ private int readInt(FileChannel fileChannel) throws IOException {+ int bytesRead = fileChannel.read(BB_4);+ assert bytesRead == 4;+ return BB_4.clear().getInt(0);+ }++ private long readLong(FileChannel fileChannel) throws IOException {+ int bytesRead = fileChannel.read(BB_8);+ assert bytesRead == 8;+ return BB_8.clear().getLong(0);+ }++ private float readFloat(FileChannel fileChannel) throws IOException {+ return Float.intBitsToFloat(readInt(fileChannel));+ }++ private double readDouble(FileChannel fileChannel) throws IOException {+ return Double.longBitsToDouble(readLong(fileChannel));+ }++ private MetadataValueType readMetadataValueType(FileChannel fileChannel) throws IOException {+ int index = readInt(fileChannel);+ return MetadataValueType.fromIndex(index);+ }++ public int getAlignment() {+ if (alignment != 0) {+ return alignment;+ }+ alignment = (int) metadata.getOrDefault("general.alignment", DEFAULT_ALIGNMENT);+ assert Integer.bitCount(alignment) == 1 : "alignment must be a power of two";+ return alignment;+ }+}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Model/ModelLoader.java b/source/net/yacy/ai/llama3/Model/ModelLoader.javanew file mode 100644index 000000000..4cd7ce40b--- /dev/null+++ b/source/net/yacy/ai/llama3/Model/ModelLoader.java@@ -0,0 +1,333 @@+/**+ * ModelLoader.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Model;++import java.io.IOException;+import java.nio.FloatBuffer;+import java.nio.channels.FileChannel;+import java.nio.file.Path;+import java.nio.file.StandardOpenOption;+import java.util.Arrays;+import java.util.List;+import java.util.Map;+import java.util.function.IntFunction;+import java.util.stream.Collectors;+import java.util.stream.IntStream;++import net.yacy.ai.llama3.Llama;+import net.yacy.ai.llama3.Tensor.FloatTensor;++public final class ModelLoader {+ private static final String TOKENIZER_LLAMA_3_MODEL = "gpt2";+ //private static final String TOKENIZER_QWEN2_7B_MODEL = "gpt2";++ private static final String LLAMA_3_PATTERN = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+";+ private final static String QWEN2_PATTERN = "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+";++ private static Vocabulary loadVocabulary(Map<String, Object> metadata) {+ String model = (String) metadata.get("tokenizer.ggml.model");+ if (!TOKENIZER_LLAMA_3_MODEL.equals(model)) {+ throw new IllegalArgumentException("expected " + TOKENIZER_LLAMA_3_MODEL + " but found " + model);+ }+ String[] tokens = (String[]) metadata.get("tokenizer.ggml.tokens");+ float[] scores = (float[]) metadata.get("tokenizer.ggml.scores");+ return new Vocabulary(tokens, scores);+ }++ public static Llama loadModel(Path ggufPath, int contextLength, boolean loadWeights) throws IOException {+ String name = ggufPath.getFileName().toString();+ if (name.contains("Llama")) return loadModelLlama3(ggufPath, contextLength, loadWeights);+ //if (name.contains("OLMo")) return loadModelLlama3(ggufPath, contextLength, loadWeights);+ if (name.contains("Qwen3")) return loadModelQwen3(ggufPath, contextLength);+ //if (name.contains("Qwen2")) return loadModelQwen2(ggufPath, contextLength);+ throw new IOException("model type unknown");+ }++ private static Llama loadModelLlama3(Path ggufPath, int contextLength, boolean loadWeights) throws IOException {+ GGUF gguf = GGUF.loadModel(ggufPath);+ FileChannel fileChannel = FileChannel.open(ggufPath, StandardOpenOption.READ);+ Map<String, Object> metadata = gguf.getMetadata();+ Vocabulary vocabulary = loadVocabulary(metadata);+ Tokenizer tokenizer = createLlama3Tokenizer(metadata, vocabulary);++ Llama.Configuration config = new Llama.Configuration(+ Arch.LLM_ARCH_LLAMA,+ (int) metadata.get("llama.embedding_length"),+ (int) metadata.get("llama.feed_forward_length"),+ (int) metadata.get("llama.block_count"),+ (int) metadata.get("llama.attention.head_count"),++ metadata.containsKey("llama.attention.head_count_kv")+ ? (int) metadata.get("llama.attention.head_count_kv")+ : (int) metadata.get("llama.attention.head_count"),++ vocabulary.size(),+ (int) metadata.get("llama.context_length"),+ false,+ (float) metadata.getOrDefault("llama.attention.layer_norm_rms_epsilon", 1e-5f),+ (float) metadata.getOrDefault("llama.rope.freq_base", 10000f)+ ).withContextLength(contextLength);++ Llama.Weights weights = null;+ if (loadWeights) {+ Map<String, GGMLTensorEntry> tensorEntries = GGUF.loadTensors(fileChannel, gguf.getTensorDataOffset(), gguf.getTensorInfos());+ weights = loadWeightsLlama3(tensorEntries, config);+ }+ return new Llama(config, tokenizer, weights);+ }++ private static Llama.Weights loadWeightsLlama3(Map<String, GGMLTensorEntry> tensorEntries, Llama.Configuration config) {+ boolean ropeScaling = tensorEntries.containsKey("rope_freqs");+ float scaleFactor = 8;+ float loFreqFactor = 1;+ float hiFreqFactor = 3;+ int oldContextLength = 8192;+ Pair<float[], float[]> ropeFreqs = precomputeFreqsCis4Llama3(config.contextLength, config.headSize, config.ropeTheta,+ ropeScaling, scaleFactor, loFreqFactor, hiFreqFactor, oldContextLength);+ float[] ropeFreqsReal = ropeFreqs.first();+ float[] ropeFreqsImag = ropeFreqs.second();++ GGMLTensorEntry tokenEmbeddings = tensorEntries.get("token_embd.weight");+ Llama.Weights qw = new Llama.Weights(+ tokenEmbeddings.loadQuantized(),+ loadArrayOfFloatBuffer(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_norm.weight")),+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_q.weight")),+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_k.weight")),+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_v.weight")),+ null, null, null,+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_output.weight")),+ loadArrayOfFloatBuffer(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_norm.weight")),+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_gate.weight")), // w1+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_down.weight")), // w2+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_up.weight")), // w3+ tensorEntries.get("output_norm.weight").toFloatBuffer(),+ FloatBuffer.wrap(ropeFreqsReal),+ FloatBuffer.wrap(ropeFreqsImag),+ // If "output.weight" is not present then the embedding weights are tied/shared with the decoder.+ // This is commonly referred as "tie word embeddings".+ tensorEntries.getOrDefault("output.weight", tokenEmbeddings).loadQuantized()+ );++ return qw;+ }++ private static Llama loadModelQwen3(Path ggufPath, int contextLength) throws IOException {+ // see https://github.com/ggml-org/llama.cpp/pull/12501/files+ GGUF gguf = GGUF.loadModel(ggufPath);+ Map<String, Object> metadata = gguf.getMetadata();++ Vocabulary vocabulary = loadVocabulary(metadata);+ Tokenizer tokenizer = createQwen2Tokenizer(metadata, vocabulary);++ int modelContextLength = (int) metadata.get("qwen2.context_length");+ if (contextLength < 0 || modelContextLength < contextLength) {+ contextLength = modelContextLength;+ }++ Llama.Configuration config = new Llama.Configuration(+ Arch.LLM_ARCH_QWEN3,+ (int) metadata.get("qwen2.embedding_length"),+ (int) metadata.get("qwen2.feed_forward_length"),+ (int) metadata.get("qwen2.block_count"),+ (int) metadata.get("qwen2.attention.head_count"),++ metadata.containsKey("qwen2.attention.head_count_kv")+ ? (int) metadata.get("qwen2.attention.head_count_kv")+ : (int) metadata.get("qwen2.attention.head_count"),++ vocabulary.size(),+ contextLength,+ false,+ (float) metadata.get("qwen2.attention.layer_norm_rms_epsilon"),+ (float) metadata.get("qwen2.rope.freq_base")+ );++ Map<String, GGMLTensorEntry> tensorEntries = gguf.getTensorEntries();++ Pair<float[], float[]> ropeFreqs = precomputeFreqsCis4Qwen2(config.contextLength, config.headSize, config.ropeTheta);+ float[] ropeFreqsReal = ropeFreqs.first();+ float[] ropeFreqsImag = ropeFreqs.second();+++ FloatTensor tokenEmbeddingTable = tensorEntries.get("token_embd.weight").loadQuantized();+ Llama.Weights qw = new Llama.Weights(+ tokenEmbeddingTable,+ loadArrayOfFloatBuffer(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_norm.weight")),+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_q.weight")),+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_k.weight")),+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_v.weight")),++ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_q.bias")),+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_k.bias")),+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_v.bias")),++ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_output.weight")),+ loadArrayOfFloatBuffer(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_norm.weight")),+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_gate.weight")), // w1+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_down.weight")), // w2+ loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_up.weight")), // w3+ tensorEntries.get("output_norm.weight").toFloatBuffer(),+ FloatBuffer.wrap(ropeFreqsReal),+ FloatBuffer.wrap(ropeFreqsImag),+ tensorEntries.containsKey("output.weight")+ ? tensorEntries.get("output.weight").loadQuantized()+ : tokenEmbeddingTable // weights are shared+ );++ return new Llama(config, tokenizer, qw);+ }++ private static Tokenizer createLlama3Tokenizer(Map<String, Object> metadata, Vocabulary vocabulary) {+ String[] mergeLines = (String[]) metadata.get("tokenizer.ggml.merges");+ List<Pair<Integer, Integer>> merges = Arrays.stream(mergeLines)+ .map(line -> line.split(" "))+ .map(parts ->+ new Pair<>(+ vocabulary.getIndex(parts[0]).orElseThrow(),+ vocabulary.getIndex(parts[1]).orElseThrow())+ ).collect(Collectors.toList());++ int allTokens = vocabulary.size();+ int baseTokens = 128000; // assume all tokens after the base ones are special.+ //int reservedSpecialTokens = allTokens - baseTokens;+ List<String> specialTokensList = Arrays.stream(vocabulary.tokens(), baseTokens, allTokens).collect(Collectors.toList());++ assert specialTokensList.stream().allMatch(token -> vocabulary.getIndex(token).isPresent());++ Map<String, Integer> specialTokens =+ IntStream.range(0, specialTokensList.size())+ .boxed()+ .collect(Collectors.toMap(+ i -> specialTokensList.get(i),+ i -> baseTokens + i)+ );++ return new Tokenizer(vocabulary, merges, LLAMA_3_PATTERN, specialTokens, null);+ }++ private static Tokenizer createQwen2Tokenizer(Map<String, Object> metadata, Vocabulary vocabulary) {+ int[] tokenTypes = (int[]) metadata.get("tokenizer.ggml.token_type");+ String[] mergeLines = (String[]) metadata.get("tokenizer.ggml.merges");+ List<Pair<Integer, Integer>> merges = Arrays.stream(mergeLines)+ .map(line -> line.split(" "))+ .map(parts ->+ new Pair<>(+ vocabulary.getIndex(parts[0]).orElseThrow(),+ vocabulary.getIndex(parts[1]).orElseThrow())+ )+ .collect(Collectors.toList());++ int allTokens = vocabulary.size();+ int baseTokens = vocabulary.getIndex("<|endoftext|>").orElseThrow(); // assume all tokens after the base ones are special.+ //int reservedSpecialTokens = allTokens - baseTokens;+ List<String> specialTokensList = Arrays.stream(vocabulary.tokens(), baseTokens, allTokens).collect(Collectors.toList());++ assert specialTokensList.stream().allMatch(token -> vocabulary.getIndex(token).isPresent());++ Map<String, Integer> specialTokens =+ IntStream.range(0, specialTokensList.size())+ .boxed()+ .collect(Collectors.toMap(+ i -> specialTokensList.get(i),+ i -> baseTokens + i)+ );++ return new Tokenizer(vocabulary, merges, QWEN2_PATTERN, specialTokens, tokenTypes);+ }+++ private static FloatTensor[] loadArrayOfQuantized(int size, IntFunction<GGMLTensorEntry> getTensorEntry) {+ FloatTensor[] array = new FloatTensor[size];+ for (int i = 0; i < size; i++) {+ array[i] = getTensorEntry.apply(i).loadQuantized();+ }+ return array;+ }++ private static FloatBuffer[] loadArrayOfFloatBuffer(int size, IntFunction<GGMLTensorEntry> getTensorEntry) {+ FloatBuffer[] array = new FloatBuffer[size];+ for (int i = 0; i < size; i++) {+ array[i] = getTensorEntry.apply(i).toFloatBuffer();+ }+ return array;+ }++ // RoPE++ // for LLama3+ private static Pair<float[], float[]> precomputeFreqsCis4Llama3(+ int contextLength, int headSize, double theta,+ boolean ropeScaling, float scaleFactor,+ float loFreqFactor, float hiFreqFactor, float oldContextLength) {+ assert headSize % 2 == 0;+ float[] cr = new float[contextLength * (headSize / 2)];+ float[] ci = new float[contextLength * (headSize / 2)];+ int n = 0;+ for (int pos = 0; pos < contextLength; ++pos) {+ for (int i = 0; i < headSize; i += 2) {+ float freq = (float) (1.0 / Math.pow(theta, i / (double) headSize));+ if (ropeScaling) {+ // Llama 3.1 scaling+ float loFreqWavelen = oldContextLength / loFreqFactor;+ float hiFreqWavelen = oldContextLength / hiFreqFactor;+ float wavelen = (float) (2.0 * Math.PI / freq);+ if (wavelen < hiFreqWavelen) {+ //freq = freq;+ } else if (wavelen > loFreqWavelen) {+ freq = freq / scaleFactor;+ } else {+ float smooth = (oldContextLength / wavelen - loFreqFactor) / (hiFreqFactor - loFreqFactor);+ freq = (1.0f - smooth) * freq / scaleFactor + smooth * freq;+ }+ }+ float val = pos * freq;+ cr[n] = (float) Math.cos(val);+ ci[n] = (float) Math.sin(val);+ n++;+ }+ }+ assert contextLength * (headSize / 2) == n;+ return new Pair<>(cr, ci);+ }++ // for Qwen3+ private static Pair<float[], float[]> precomputeFreqsCis4Qwen2(+ int contextLength, int headSize, double theta) {+ assert headSize % 2 == 0;+ float[] cr = new float[contextLength * (headSize / 2)];+ float[] ci = new float[contextLength * (headSize / 2)];+ int n = 0;+ for (int pos = 0; pos < contextLength; ++pos) {+ for (int i = 0; i < headSize; i += 2) {+ float freq = (float) (1.0 / Math.pow(theta, i / (double) headSize));+ float val = pos * freq;+ cr[n] = (float) Math.cos(val);+ ci[n] = (float) Math.sin(val);+ n++;+ }+ }+ assert contextLength * (headSize / 2) == n;+ return new Pair<>(cr, ci);+ }++}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Model/Pair.java b/source/net/yacy/ai/llama3/Model/Pair.javanew file mode 100644index 000000000..528994cc8--- /dev/null+++ b/source/net/yacy/ai/llama3/Model/Pair.java@@ -0,0 +1,59 @@+/**+ * Pair.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Model;++import java.util.Objects;+++public final class Pair<F, S> {+ public final F first;+ public final S second;++ public Pair(F first, S second) {+ this.first = first;+ this.second = second;+ }++ public F first() { return first; }+ public S second() { return second; }++ // Optional factory method (avoids 'new' keyword)+ public static <F, S> Pair<F, S> of(F first, S second) {+ return new Pair<>(first, second);+ }++ @Override+ public boolean equals(Object o) {+ if (this == o) return true;+ if (!(o instanceof Pair)) return false;+ Pair<?, ?> pair = (Pair<?, ?>) o;+ return Objects.equals(first, pair.first) &&+ Objects.equals(second, pair.second);+ }++ @Override+ public int hashCode() {+ return Objects.hash(first, second);+ }++}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Model/Tokenizer.java b/source/net/yacy/ai/llama3/Model/Tokenizer.javanew file mode 100644index 000000000..8b3a118a1--- /dev/null+++ b/source/net/yacy/ai/llama3/Model/Tokenizer.java@@ -0,0 +1,295 @@+/**+ * Tokenizer.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Model;++import java.util.ArrayList;+import java.util.Arrays;+import java.util.Comparator;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.regex.Matcher;+import java.util.regex.Pattern;+import java.util.stream.Collectors;+import java.util.stream.IntStream;+import java.nio.charset.StandardCharsets;++/**+ * Byte Pair Encoding tokenizer.+ * <p>+ * Based on <a href="https://github.com/karpathy/minbpe">minbpe</a>, algorithmically follows along the+ * <a href="https://github.com/openai/gpt-2/blob/master/src/encoder.py">GPT 2 tokenizer</a>+ */+public class Tokenizer {+ private final Pattern compiledPattern;+ private final Vocabulary vocabulary;+ private final Map<Pair<Integer, Integer>, Integer> merges;+ private final Map<String, Integer> specialTokens;+ private final int[] tokenTypes;++ public String regexPattern() {+ if (compiledPattern == null) {+ return null;+ }+ return compiledPattern.pattern();+ }++ public Map<String, Integer> getSpecialTokens() {+ return specialTokens;+ }++ public boolean isSpecialToken(int tokenIndex) {+ return specialTokens.containsValue(tokenIndex);+ }++ public int getTokenType(int tokenIndex) {+ return tokenTypes[tokenIndex];+ }++ public Tokenizer(Vocabulary vocabulary, List<Pair<Integer, Integer>> merges, String regexPattern, Map<String, Integer> specialTokens, int[] tokenTypes) {+ this.vocabulary = vocabulary;+ this.compiledPattern = regexPattern != null ? Pattern.compile(regexPattern) : null;+ this.specialTokens = new HashMap<>(specialTokens);+ this.merges = new HashMap<>();+ this.tokenTypes = tokenTypes;+ for (Pair<Integer, Integer> pair : merges) {+ int firstIndex = pair.first();+ int secondIndex = pair.second();+ int mergeIndex = vocabulary.getIndex(vocabulary.get(firstIndex) + vocabulary.get(secondIndex)).orElseThrow();+ this.merges.put(pair, mergeIndex);+ }+ }++ private int[] encodeImpl(String text) {+ return encode(text, Set.of()).stream().mapToInt(i -> i).toArray();+ }++ /**+ * Unlike {@link #encodeOrdinary(String)}, this function handles special tokens.+ * allowed_special: can be "all"|"none"|"none_raise" or a custom set of special tokens+ * if none_raise, then an error is raised if any special token is encountered in text+ * this is the default tiktoken behavior right now as well+ * any other behavior is either annoying, or a major footgun.+ */+ List<Integer> encode(String text, Set<String> allowedSpecial) {+ // decode the user desire w.r.t. handling of special tokens+ Set<String> special = allowedSpecial;+ assert getSpecialTokens().keySet().containsAll(special);+ if (special.isEmpty()) {+ // shortcut: if no special tokens, just use the ordinary encoding+ return encodeOrdinary(text);+ }++ // otherwise, we have to be careful with potential special tokens in text+ // we handle special tokens by splitting the text+ // based on the occurrence of any exact match with any of the special tokens+ // we can use re.split for this. note that surrounding the pattern with ()+ // makes it into a capturing group, so the special tokens will be included+ String specialPattern = special+ .stream()+ .map(Pattern::quote)+ .collect(Collectors.joining("|", "(", ")"));++ String[] specialChunks = text.split(specialPattern);+ // now all the special characters are separated from the rest of the text+ // all chunks of text are encoded separately, then results are joined+ List<Integer> ids = new ArrayList<>();+ for (String part : specialChunks) {+ if (special.contains(part)) {+ // this is a special token, encode it separately as a special case+ ids.add(getSpecialTokens().get(part));+ } else {+ // this is an ordinary sequence, encode it normally+ ids.addAll(encodeOrdinary(part));+ }+ }+ return ids;+ }++ private static List<String> findAll(Pattern pattern, String text) {+ List<String> allMatches = new ArrayList<>();+ Matcher matcher = pattern.matcher(text);+ while (matcher.find()) {+ allMatches.add(matcher.group());+ }+ return allMatches;+ }++ /**+ * Encoding that ignores any special tokens.+ */+ public List<Integer> encodeOrdinary(String text) {+ // split text into chunks of text by categories defined in regex pattern+ List<String> textChunks = findAll(compiledPattern, text);+ // all chunks of text are encoded separately, then results are joined+ List<Integer> ids = new ArrayList<>();+ for (String chunk : textChunks) {+ List<Integer> chunkIds = encodeChunk(chunk);+ ids.addAll(chunkIds);+ }+ return ids;+ }++ private Map<Pair<Integer, Integer>, Integer> getStats(List<Integer> ids) {+ Map<Pair<Integer, Integer>, Integer> map = new HashMap<>();+ for (int i = 0; i + 1 < ids.size(); i++) {+ Pair<Integer, Integer> key = new Pair<>(ids.get(i), ids.get(i + 1));+ map.put(key, map.getOrDefault(key, 0) + 1);+ }+ return map;+ }++ private List<Integer> encodeChunk(String chunk) {+ // return the token ids+ // let's begin. first, convert all bytes to integers in range 0..255+ List<Integer> ids = new ArrayList<>();+ for (int b : chunk.toCharArray()) {+ int tokenIndex = this.vocabulary.getIndex(String.valueOf((char) b)).orElseThrow();+ ids.add(tokenIndex);+ }++ while (ids.size() >= 2) {+ // find the pair with the lowest merge index+ Map<Pair<Integer, Integer>, Integer> stats = getStats(ids);+ Pair<Integer, Integer> pair = stats.keySet().stream().min(Comparator.comparingInt(key -> this.merges.getOrDefault(key, Integer.MAX_VALUE))).orElseThrow();+ // subtle: if there are no more merges available, the key will+ // result in an inf for every single pair, and the min will be+ // just the first pair in the list, arbitrarily+ // we can detect this terminating case by a membership check+ if (!this.merges.containsKey(pair)) {+ break; // nothing else can be merged anymore+ }+ // otherwise let's merge the best pair (lowest merge index)+ int idx = this.merges.get(pair);+ ids = merge(ids, pair, idx);+ }+ return ids;+ }++ private static List<Integer> merge(List<Integer> ids, Pair<Integer, Integer> pair, int idx) {+ List<Integer> newids = new ArrayList<>();+ int i = 0;+ while (i < ids.size()) {+ // if not at the very last position AND the pair matches, replace it+ if (ids.get(i).equals(pair.first()) && i < ids.size() - 1 && ids.get(i + 1).equals(pair.second())) {+ newids.add(idx);+ i += 2;+ } else {+ newids.add(ids.get(i));+ i += 1;+ }+ }+ return newids;+ }++ public String decodeImpl(List<Integer> tokens) {+ StringBuilder sb = new StringBuilder();+ for (int token : tokens) {+ String tokenString = vocabulary.get(token);+ sb.append(tokenString);+ }+ return sb.toString();+ }++ /**+ * Returns list of utf-8 byte and a corresponding list of unicode strings.+ * The reversible bpe codes work on unicode strings.+ * This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.+ * When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.+ * This is a significant percentage of your normal, say, 32K bpe vocab.+ * To avoid that, we want lookup tables between utf-8 bytes and unicode strings.+ * And avoids mapping to whitespace/control characters the bpe code barfs on.+ */+ private static Map<Integer, Integer> bytesToUnicode() {+ List<Integer> bs = new ArrayList<>();+ IntStream.rangeClosed('!', '~').forEach(bs::add);+ IntStream.rangeClosed('¡', '¬').forEach(bs::add);+ IntStream.rangeClosed('®', 'ÿ').forEach(bs::add);++ List<Integer> cs = new ArrayList<>(bs);+ int n = 0;+ for (int b = 0; b < 256; ++b) {+ if (!bs.contains(b)) {+ bs.add(b);+ cs.add(256 + n);+ n += 1;+ }+ }++ // return dict(zip(bs, cs))+ return IntStream.range(0, bs.size())+ .boxed()+ .collect(Collectors.toMap(bs::get, cs::get));+ }++ static final Map<Integer, Integer> BYTE_ENCODER = bytesToUnicode();+ static final Map<Integer, Integer> BYTE_DECODER = BYTE_ENCODER.entrySet()+ .stream()+ .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey));++ public int[] encode(String text) {+ StringBuilder sb = new StringBuilder();+ byte[] bytes = text.getBytes(StandardCharsets.UTF_8);+ for (byte b : bytes) {+ sb.appendCodePoint(BYTE_ENCODER.get(Byte.toUnsignedInt(b)));+ }+ return encodeImpl(sb.toString());+ }++ /*+ public static String replaceControlCharacters(int[] codePoints) {+ // we don't want to print control characters+ // which distort the output (e.g. \n or much worse)+ // https://stackoverflow.com/questions/4324790/removing-control-characters-from-a-string-in-python/19016117#19016117+ // http://www.unicode.org/reports/tr44/#GC_Values_Table\+ StringBuilder chars = new StringBuilder();+ for (int cp : codePoints) {+ if (Character.getType(cp) == Character.CONTROL && cp != '\n') {+ chars.append("\\u").append(HexFormat.of().toHexDigits(cp, 4)); // escape+ } else {+ chars.appendCodePoint(cp); // this character is ok+ }+ }+ return chars.toString();+ }++ public static String replaceControlCharacters(String str) {+ return replaceControlCharacters(str.codePoints().toArray());+ }+ */++ public List<Integer> encodeAsList(String text) {+ return Arrays.stream(encode(text)).boxed().collect(Collectors.toList());+ }++ public String decode(List<Integer> tokens) {+ String decoded = decodeImpl(tokens);+ int[] decodedBytesAsInts = decoded.codePoints().map(BYTE_DECODER::get).toArray();+ byte[] rawBytes = new byte[decodedBytesAsInts.length];+ for (int i = 0; i < decoded.length(); i++) {+ rawBytes[i] = (byte) decodedBytesAsInts[i];+ }+ return new String(rawBytes, StandardCharsets.UTF_8);+ }+}diff --git a/source/net/yacy/ai/llama3/Model/Vocabulary.java b/source/net/yacy/ai/llama3/Model/Vocabulary.javanew file mode 100644index 000000000..f2ca7e4dd--- /dev/null+++ b/source/net/yacy/ai/llama3/Model/Vocabulary.java@@ -0,0 +1,65 @@+/**+ * Vocabulary.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Model;++import java.util.*;+import java.util.stream.*;++public final class Vocabulary {+ private final String[] tokens;+ private final float[] scores;+ private final Map<String, Integer> tokenToIndex;++ // Primary constructor+ public Vocabulary(String[] tokens, float[] scores, Map<String, Integer> tokenToIndex) {+ this.tokens = tokens == null ? null : Arrays.copyOf(tokens, tokens.length);+ this.scores = scores == null ? null : Arrays.copyOf(scores, scores.length);+ this.tokenToIndex = tokenToIndex == null ? null : new HashMap<>(tokenToIndex);+ }++ // Secondary constructor+ public Vocabulary(String[] vocabulary, float[] scores) {+ this(vocabulary, scores,+ IntStream.range(0, vocabulary.length)+ .boxed()+ .collect(Collectors.toMap(i -> vocabulary[i], i -> i))+ );+ }++ public String[] tokens() {+ return Arrays.copyOf(tokens, tokens.length);+ }++ public String get(int tokenIndex) {+ return tokens[tokenIndex];+ }++ public OptionalInt getIndex(String token) {+ Integer value = tokenToIndex.get(token);+ return value != null ? OptionalInt.of(value) : OptionalInt.empty();+ }++ public int size() {+ return tokens.length;+ }+}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Sampler.java b/source/net/yacy/ai/llama3/Sampler.javanew file mode 100644index 000000000..c94876cab--- /dev/null+++ b/source/net/yacy/ai/llama3/Sampler.java@@ -0,0 +1,175 @@+/**+ * Sampler.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3;++import java.util.Comparator;+import java.util.Random;++import net.yacy.ai.llama3.Tensor.FloatTensor;++@FunctionalInterface+interface Sampler {+ int sampleToken(FloatTensor logits);++ Sampler ARGMAX = FloatTensor::argmax;++ static Sampler selectSampler(int vocabularySize, float temperature, float topp, long rngSeed) {+ Sampler sampler;+ if (temperature == 0.0f) {+ // greedy argmax sampling: take the token with the highest probability+ sampler = Sampler.ARGMAX;+ } else {+ // we sample from this distribution to get the next token+ // RandomGeneratorFactory.getDefault().create(rngSeed); requires additonal native-image configuration.+ Random rng = new Random(rngSeed);+ Sampler innerSampler;+ if (topp <= 0 || topp >= 1) {+ // simply sample from the predicted probability distribution+ innerSampler = new CategoricalSampler(rng);+ } else {+ // top-p (nucleus) sampling, clamping the least likely tokens to zero+ innerSampler = new ToppSampler(vocabularySize, topp, rng);+ }+ sampler = logits -> {+ // apply the temperature to the logits+ logits.divideInPlace(0, logits.size(), temperature);+ // apply softmax to the logits to get the probabilities for next token+ logits.softmaxInPlace(0, logits.size());+ return innerSampler.sampleToken(logits);+ };+ }+ return sampler;+ }++ static class CategoricalSampler implements Sampler {++ final Random rng;++ public CategoricalSampler(Random rng) {+ this.rng = rng;+ }++ @Override+ public int sampleToken(FloatTensor logits) {+ // sample index from probabilities (they must sum to 1!)+ float random0to1 = rng.nextFloat();+ float cdf = 0.0f;+ for (int i = 0; i < logits.size(); i++) {+ cdf += logits.getFloat(i);+ if (random0to1 < cdf) {+ return i;+ }+ }+ return logits.size() - 1; // in case of rounding errors+ }+ }++ static class ToppSampler implements Sampler {++ final int[] indices;+ final float topp;+ final Random rng;++ public ToppSampler(int maxNumberOfElements, float topp, Random rng) {+ this.indices = new int[maxNumberOfElements];+ this.topp = topp;+ this.rng = rng;+ }++ static void swap(int[] array, int from, int to) {+ int tmp = array[from];+ array[from] = array[to];+ array[to] = tmp;+ }++ static void siftDown(int[] array, int from, int n, Comparator<Integer> comparator) {+ int prev = from, next;+ while ((next = 2 * prev + 1) < n) {+ int r = 2 * prev + 2;+ if (r < n && comparator.compare(array[r], array[next]) < 0) {+ next = r;+ }+ if (comparator.compare(array[next], array[prev]) < 0) {+ swap(array, prev, next);+ prev = next;+ } else {+ break;+ }+ }+ }++ @Override+ public int sampleToken(FloatTensor logits) {+ // top-p sampling (or "nucleus sampling") samples from the smallest set of+ // tokens that exceed probability topp. This way we never sample tokens that+ // have very low probabilities and are less likely to go "off the rails".+ Comparator<Integer> comparator = Comparator.comparingDouble(logits::getFloat).reversed();++ int n = logits.size();+ int head = 0;+ int tail = n - 1;+ // values smaller than (1 - topp) / (n - 1) cannot be part of the result+ // so for efficiency we crop these out as candidates before sorting+ float cutoff = (1.0f - topp) / (n - 1);+ for (int i = 0; i < indices.length; i++) {+ if (logits.getFloat(i) >= cutoff) {+ indices[head++] = i;+ } else {+ indices[tail--] = i;+ }+ }++ int n0 = head;+ // build heap O(n0)+ for (int i = n0 / 2 - 1; i >= 0; --i) {+ siftDown(indices, i, n0, comparator);+ }++ // truncate the list where cumulative probability of the largest k elements exceeds topp+ // O(k lg n0)+ float cumulativeProb = 0.0f;+ int lastIndex = 0;+ for (int i = n0 - 1; i >= 0; i--) {+ swap(indices, 0, i);+ cumulativeProb += logits.getFloat(indices[i]);+ if (cumulativeProb > topp) {+ lastIndex = i;+ break; // we've exceeded topp by including lastIndex+ }+ siftDown(indices, 0, i - 1, comparator);+ }++ // sample from the truncated list+ float r = rng.nextFloat() * cumulativeProb;+ float cdf = 0.0f;+ for (int i = n0 - 1; i >= lastIndex; i--) {+ cdf += logits.getFloat(indices[i]);+ if (r < cdf) {+ return indices[i];+ }+ }++ return indices[lastIndex]; // in case of rounding errors+ }+ }+}diff --git a/source/net/yacy/ai/llama3/Tensor/ArrayFloatTensor.java b/source/net/yacy/ai/llama3/Tensor/ArrayFloatTensor.javanew file mode 100644index 000000000..9e0155eba--- /dev/null+++ b/source/net/yacy/ai/llama3/Tensor/ArrayFloatTensor.java@@ -0,0 +1,177 @@+/**+ * ArrayFloatTensor.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Tensor;++import java.util.Arrays;+import java.lang.invoke.MethodHandles;+import java.lang.invoke.VarHandle;++import net.yacy.ai.llama3.Model.GGMLType;++public final class ArrayFloatTensor extends FloatTensor {++ public final float[] values;+ private static final VarHandle FLOAT_ARRAY_HANDLE;++ static {+ try {+ FLOAT_ARRAY_HANDLE = MethodHandles.arrayElementVarHandle(float[].class);+ } catch (Exception e) {+ throw new RuntimeException("Failed to get VarHandle for float[]", e);+ }+ }++ public ArrayFloatTensor(final float[] values) {+ this.values = values;+ }++ public static FloatTensor allocate(final int... dims) {+ int numberOfElements = FloatTensor.numberOfElements(dims);+ return new ArrayFloatTensor(new float[numberOfElements]);+ }++ @Override+ public final int size() {+ return this.values.length;+ }++ @Override+ public final float getFloat(final int index) {+ return (float) FLOAT_ARRAY_HANDLE.get(this.values, index);+ }++ @Override+ public final void setFloat(final int index, final float value) {+ FLOAT_ARRAY_HANDLE.set(this.values, index, value);+ }++ @Override+ public GGMLType type() {+ return GGMLType.F32;+ }++ @Override+ public final FloatTensor fillInPlace(final int thisOffset, final int size, final float value) {+ Arrays.fill(this.values, thisOffset, thisOffset + size, value);+ return this;+ }++ @Override+ public final FloatTensor mapInPlace(final int thisOffset, final int size, MapFunction mapFunction) {+ int endIndex = thisOffset + size;+ for (int i = thisOffset; i < endIndex; ++i) {+ this.values[i] = mapFunction.apply(this.values[i]);+ }+ return this;+ }++ @Override+ public final void copyTo(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {+ final int delta = thisOffset - thatOffset;+ if (that instanceof ArrayFloatTensor) {+ final ArrayFloatTensor aft = (ArrayFloatTensor) that;+ int endOffset = thatOffset + size;+ for (int i = thatOffset; i < endOffset; ++i) {+ FLOAT_ARRAY_HANDLE.set(aft.values, i, (float) FLOAT_ARRAY_HANDLE.get(this.values, i + delta));+ }+ } else {+ int endOffset = thatOffset + size;+ for (int i = thatOffset; i < endOffset; ++i) {+ that.setFloat(i, (float) FLOAT_ARRAY_HANDLE.get(this.values, i + delta));+ }+ }+ }++ @Override+ public final float dot(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {+ float result = 0f;+ if (that instanceof ArrayFloatTensor) {+ final ArrayFloatTensor aft = (ArrayFloatTensor) that;+ final float[] a = this.values;+ final float[] b = aft.values;++ for (int i = 0; i < size; i++) {+ final float valA = (float) FLOAT_ARRAY_HANDLE.get(a, thisOffset + i);+ final float valB = (float) FLOAT_ARRAY_HANDLE.get(b, thatOffset + i);+ result += valA * valB;+ }+ } else {+ final float[] a = this.values;+ for (int i = 0; i < size; i++) {+ final float valA = (float) FLOAT_ARRAY_HANDLE.get(a, thisOffset + i);+ result += valA * that.getFloat(thatOffset + i);+ }+ }+ return result;+ }++ @Override+ public final void matmul(final FloatTensor that, final FloatTensor out, final int dim0, final int dim1) {+ if (that instanceof ArrayFloatTensor) {+ parallelFor(0, dim0, i -> ((ArrayFloatTensor) out).values[i] = this.dot(i * dim1, that, 0, dim1));+ } else {+ parallelFor(0, dim0, i -> out.setFloat(i, this.dot(i * dim1, that, 0, dim1)));+ }+ }++ @Override+ public final void matmul(final int context, final FloatTensor[] that, final FloatTensor[] out, final int dim0, final int dim1) {+ if (that.length != out.length) {+ throw new IllegalArgumentException(String.format("that.len=%d, out.len=%d", that.length, out.length));+ }+ parallelForLong(0, dim0 * context, ti -> {+ int idxArr = (int) (ti / dim0);+ int i = (int) (ti % dim0);+ out[idxArr].setFloat(i, this.dot(i * dim1, that[idxArr], 0, dim1));+ });+ }++ @Override+ public final FloatTensor saxpyInPlace(final int thisOffset, final FloatTensor that, final int thatOffset, final int size, final float a) {+ if (that instanceof Q4_0FloatTensor) {+ Q4_0FloatTensor qft = (Q4_0FloatTensor) that;+ final float[] decodedBlock = Q4_0FloatTensor.scratchBuffer.get();+ int remaining = size;+ int i = 0;++ while (remaining > 0) {+ int chunkSize = Math.min(remaining, GGMLType.Q4_0.blockSize);+ qft.getFloatArray(thatOffset + i, decodedBlock, 0, chunkSize);+ for (int j = 0; j < chunkSize; ++j) {+ int dstIdx = thisOffset + i + j;+ this.setFloat(dstIdx, a * decodedBlock[j] + this.getFloat(dstIdx));+ }+ i += chunkSize;+ remaining -= chunkSize;+ }+ } else {+ for (int i = 0; i < size; ++i) {+ int idx = thisOffset + i;+ this.setFloat(idx, a * that.getFloat(thatOffset + i) + this.getFloat(idx));+ }+ }++ return this;+ }++}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Tensor/BF16FloatTensor.java b/source/net/yacy/ai/llama3/Tensor/BF16FloatTensor.javanew file mode 100644index 000000000..0de5ac525--- /dev/null+++ b/source/net/yacy/ai/llama3/Tensor/BF16FloatTensor.java@@ -0,0 +1,74 @@+/**+ * BF16FloatTensor.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Tensor;++import java.nio.ByteBuffer;+import java.nio.ByteOrder;++import net.yacy.ai.llama3.Model.GGMLType;++public final class BF16FloatTensor extends FloatTensor {++ final int size;+ final ByteBuffer buffer;++ public BF16FloatTensor(int size, ByteBuffer buffer) {+ if (buffer.remaining() < size * GGMLType.BFLOAT16_BYTES) {+ throw new IllegalArgumentException("Buffer too small");+ }+ this.size = size;+ this.buffer = buffer.duplicate().order(ByteOrder.nativeOrder());+ }++ public BF16FloatTensor(final float[] values) {+ this.size = values.length;+ this.buffer = ByteBuffer.allocateDirect(size * GGMLType.BFLOAT16_BYTES).order(ByteOrder.nativeOrder());+ for (int i = 0; i < size; i++) {+ setFloat(i, values[i]);+ }+ }++ @Override+ public int size() {+ return size;+ }++ @Override+ public GGMLType type() {+ return GGMLType.BF16;+ }++ @Override+ public final void setFloat(int index, float value) {+ assert 0 <= index && index < size;+ int hBits = Float.floatToIntBits(value) >>> 16; // convert float to bfloat16+ buffer.putShort(index * GGMLType.BFLOAT16_BYTES, (short) hBits);+ }++ @Override+ public final float getFloat(int index) {+ assert 0 <= index && index < size;+ return Float.intBitsToFloat(buffer.getShort(index * GGMLType.BFLOAT16_BYTES) << 16);+ }++}diff --git a/source/net/yacy/ai/llama3/Tensor/DirectBufferFloatTensor.java b/source/net/yacy/ai/llama3/Tensor/DirectBufferFloatTensor.javanew file mode 100644index 000000000..64c615fec--- /dev/null+++ b/source/net/yacy/ai/llama3/Tensor/DirectBufferFloatTensor.java@@ -0,0 +1,111 @@+/**+ * DirectBufferFloatTensor+ * Copyright 2025 by Michael Peter Christen+ * First released 19.06.2025 at https://yacy.net+ *+ ** This class was not part of the original llama3 implementation,+ ** but added later by the author to support different architectures.+ ** It therefore does not inherit the llama3 copyright.+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.llama3.Tensor;++import java.nio.ByteBuffer;+import java.nio.ByteOrder;+import java.nio.FloatBuffer;++import net.yacy.ai.llama3.Model.GGMLType;++public class DirectBufferFloatTensor extends FloatTensor {++ final FloatBuffer floatBuffer;++ public DirectBufferFloatTensor(ByteBuffer byteBuffer) {+ if (byteBuffer.isDirect()) {+ this.floatBuffer = byteBuffer.asFloatBuffer();+ } else {+ int capacityBytes = byteBuffer.remaining();+ ByteBuffer directByteBuffer = ByteBuffer.allocateDirect(capacityBytes).order(byteBuffer.order());+ directByteBuffer.put(byteBuffer.duplicate());+ directByteBuffer.flip();+ this.floatBuffer = directByteBuffer.asFloatBuffer();+ }+ }++ public DirectBufferFloatTensor(final float[] values) {+ int capacityBytes = values.length * Float.BYTES;+ ByteBuffer directByteBuffer = ByteBuffer.allocateDirect(capacityBytes).order(ByteOrder.nativeOrder());+ this.floatBuffer = directByteBuffer.asFloatBuffer();+ this.floatBuffer.put(values);+ }++ public static FloatTensor allocate(final int... dims) {+ int numberOfElements = FloatTensor.numberOfElements(dims);+ int bytesNeeded = numberOfElements * Float.BYTES;+ ByteBuffer buffer = ByteBuffer.allocateDirect(bytesNeeded).order(ByteOrder.nativeOrder());+ return new DirectBufferFloatTensor(buffer);+ }++ @Override+ public final int size() {+ return this.floatBuffer.capacity();+ }++ @Override+ public final float getFloat(final int index) {+ return this.floatBuffer.get(index);+ }++ @Override+ public final void setFloat(final int index, final float value) {+ this.floatBuffer.put(index, value);+ }++ @Override+ public final GGMLType type() {+ return GGMLType.F32;+ }++ @Override+ public final float dot(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {+ float result = 0f;+ for (int j = 0; j < size; j++) {+ result += this.floatBuffer.get(thisOffset + j) * that.getFloat(thatOffset + j);+ }+ return result;+ }++ @Override+ public final FloatTensor fillInPlace(final int thisOffset, final int size, final float value) {+ int end = thisOffset + size;+ for (int i = thisOffset; i < end; i++) {+ floatBuffer.put(i, value);+ }+ return this;+ }++ @Override+ public final FloatTensor mapInPlace(final int thisOffset, final int size, MapFunction mapFunction) {+ int end = thisOffset + size;+ for (int i = thisOffset; i < end; i++) {+ float current = floatBuffer.get(i);+ floatBuffer.put(i, mapFunction.apply(current));+ }+ return this;+ }++}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Tensor/F16FloatTensor.java b/source/net/yacy/ai/llama3/Tensor/F16FloatTensor.javanew file mode 100644index 000000000..d41fdbc87--- /dev/null+++ b/source/net/yacy/ai/llama3/Tensor/F16FloatTensor.java@@ -0,0 +1,75 @@+/**+ * F16FloatTensor.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */+++package net.yacy.ai.llama3.Tensor;++import java.nio.ByteBuffer;+import java.nio.ByteOrder;++import net.yacy.ai.llama3.Model.GGMLType;++public final class F16FloatTensor extends FloatTensor {++ final int size;+ final ByteBuffer buffer;++ public F16FloatTensor(int size, ByteBuffer buffer) {+ if (buffer.remaining() < size * GGMLType.FLOAT16_BYTES) {+ throw new IllegalArgumentException("Buffer too small");+ }+ this.size = size;+ this.buffer = buffer.duplicate().order(ByteOrder.nativeOrder());+ }++ public F16FloatTensor(final float[] values) {+ this.size = values.length;+ this.buffer = ByteBuffer.allocateDirect(size * GGMLType.FLOAT16_BYTES).order(ByteOrder.nativeOrder());+ for (int i = 0; i < size; i++) {+ setFloat(i, values[i]);+ }+ }++ @Override+ public int size() {+ return size;+ }++ @Override+ public GGMLType type() {+ return GGMLType.F16;+ }++ @Override+ public final void setFloat(int index, float value) {+ assert 0 <= index && index < size;+ short hBits = floatToFloat16(value);+ buffer.putShort(index * GGMLType.FLOAT16_BYTES, hBits);+ }++ @Override+ public final float getFloat(int index) {+ assert 0 <= index && index < size;+ return float16ToFloat(buffer.getShort(index * GGMLType.FLOAT16_BYTES));+ }++}diff --git a/source/net/yacy/ai/llama3/Tensor/FloatTensor.java b/source/net/yacy/ai/llama3/Tensor/FloatTensor.javanew file mode 100644index 000000000..4457b10d0--- /dev/null+++ b/source/net/yacy/ai/llama3/Tensor/FloatTensor.java@@ -0,0 +1,286 @@+/**+ * FloatTensor.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Tensor;++import java.util.Arrays;+import java.util.function.IntConsumer;+import java.util.function.LongConsumer;+import java.util.stream.IntStream;+import java.util.stream.LongStream;++import net.yacy.ai.llama3.Model.GGMLType;++/**+ * Over-simplified, shapeless, float tensor.+ * <p>+ * Not a strict tensor, but rather just a sequence of floats, not required to be backed by memory+ * e.g. can represent a sequence of quantized floats.+ */+public abstract class FloatTensor {++ /**+ * Converts a 16-bit float (half-precision) to a 32-bit float (single-precision).+ *+ * @param h the half-precision float as a short+ * @return the single-precision float+ */+ public final static float float16ToFloat(short h) {++ final int hBits = h & 0xFFFF; // treat as unsigned+ final int sign = (hBits >>> 15) & 0x00000001;++ int exp = (hBits >>> 10) & 0x0000001F;+ int mant = hBits & 0x000003FF;+ int fBits;++ if (exp == 0) {+ if (mant == 0) {+ // zero+ fBits = sign << 31;+ } else {+ // subnormal+ while ((mant & 0x00000400) == 0) {+ mant <<= 1;+ exp -= 1;+ }+ exp += 1;+ mant &= ~0x00000400;+ fBits = (sign << 31) | ((exp + 127 - 15) << 23) | (mant << 13);+ }+ } else if (exp == 31) {+ // Inf/NaN+ fBits = (sign << 31) | 0x7F800000 | (mant << 13);+ } else {+ // normalized number+ fBits = (sign << 31) | ((exp + 127 - 15) << 23) | (mant << 13);+ }++ return Float.intBitsToFloat(fBits);+ }++ /**+ * Converts a 32-bit float (single-precision) to a 16-bit float (half-precision).+ *+ * @param f the single-precision float+ * @return the half-precision float as a short+ */+ public final static short floatToFloat16(final float f) {+ final int fBits = Float.floatToIntBits(f);+ final int sign = (fBits >>> 31) & 0x00000001;+ final int exp = (fBits >>> 23) & 0x000000FF;+ final int mant = fBits & 0x007FFFFF;++ short hBits;++ if (exp == 0xFF) {+ // Inf/NaN+ hBits = (short) ((sign << 15) | 0x7C00 | (mant >>> 13));+ } else if (exp < 112) {+ // subnormal or zero+ hBits = (short) (sign << 15);+ } else if (exp > 143) {+ // overflow to Inf+ hBits = (short) ((sign << 15) | 0x7C00);+ } else {+ // normalized number+ hBits = (short) ((sign << 15) | ((exp - 112) << 10) | (mant >>> 13));+ }++ return hBits;+ }++ public static void parallelFor(final int startInclusive, final int endExclusive, final IntConsumer action) {+ if (startInclusive == 0 && endExclusive == 1) {+ action.accept(0);+ return;+ }+ IntStream.range(startInclusive, endExclusive).parallel().forEach(action);+ }++ public static void parallelForLong(final long startInclusive, final long endExclusive, final LongConsumer action) {+ if (startInclusive == 0 && endExclusive == 1) {+ action.accept(0);+ return;+ }+ LongStream.range(startInclusive, endExclusive).parallel().forEach(action);+ }++ public abstract int size();++ public abstract float getFloat(final int index);++ public abstract void setFloat(final int index, final float value);++ abstract GGMLType type();++ public static int numberOfElements(final int... dimensions) {+ assert Arrays.stream(dimensions).allMatch(i -> i > 0);+ return Arrays.stream(dimensions).reduce(Math::multiplyExact).orElseThrow();+ }++ public static float scalarDot(final FloatTensor thiz, final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {+ float result = 0f;+ for (int j = 0; j < size; j++) {+ result += thiz.getFloat(thisOffset + j) * that.getFloat(thatOffset + j);+ }+ return result;+ }++ public float dot(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {+ return scalarDot(this, thisOffset, that, thatOffset, size);+ }++ public void matmul(final FloatTensor that, final FloatTensor out, final int dim0, final int dim1) {+ parallelFor(0, dim0, i -> out.setFloat(i, dot(i * dim1, that, 0, dim1)));+ }++ public void matmul(final int context, final FloatTensor[] that, final FloatTensor[] out, final int dim0, final int dim1) {+ if (that.length != out.length) {+ throw new IllegalArgumentException(String.format("that.len=%d, out.len=%d", that.length, out.length));+ }+ parallelForLong(0, dim0 * context, ti -> {+ int idxArr = (int) (ti / dim0);+ int i = (int) (ti % dim0);+ out[idxArr].setFloat(i, dot(i * dim1, that[idxArr], 0, dim1));+ });+ }++ @FunctionalInterface+ public interface AggregateFunction {+ float apply(float acc, float value);+ }++ public float reduce(final int thisOffset, final int size, final float seed, final AggregateFunction reduce) {+ float result = seed;+ for (int i = 0; i < size; ++i) {+ result = reduce.apply(result, getFloat(thisOffset + i));+ }+ return result;+ }++ private float sum(final int thisOffset, final int size) {+ return reduce(thisOffset, size, 0f, Float::sum);+ }++ private float max(final int thisOffset, final int size) {+ return reduce(thisOffset, size, Float.NEGATIVE_INFINITY, Float::max);+ }++ public void copyTo(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {+ int endOffset = thatOffset + size;+ for (int i = thatOffset; i < endOffset; ++i) {+ that.setFloat(i, this.getFloat(i - thatOffset + thisOffset));+ }+ }++ private int argmax(final int thisOffset, final int size) {+ assert size > 0;+ int maxIndex = thisOffset;+ float maxValue = this.getFloat(maxIndex);+ int endIndex = thisOffset + size;+ for (int i = thisOffset; i < endIndex; ++i) {+ float f = this.getFloat(i);+ if (f > maxValue) {+ maxValue = f;+ maxIndex = i;+ }+ }+ return maxIndex;+ }++ public int argmax() {+ return argmax(0, size());+ }++ @FunctionalInterface+ public interface MapFunction {+ float apply(float value);+ }++ @FunctionalInterface+ public interface MapWithIndexFunction {+ float apply(float value, int index);+ }++ public FloatTensor mapInPlace(final int thisOffset, final int size, MapFunction mapFunction) {+ int endIndex = thisOffset + size;+ for (int i = thisOffset; i < endIndex; ++i) {+ setFloat(i, mapFunction.apply(getFloat(i)));+ }+ return this;+ }++ public final FloatTensor mapInPlace(final MapFunction mapFunction) {+ return mapInPlace(0, size(), mapFunction);+ }++ public final FloatTensor mapWithIndexInPlace(final int thisOffset, final int size, final FloatTensor.MapWithIndexFunction mapWithIndexFunction) {+ int endOffset = thisOffset + size;+ for (int i = thisOffset; i < endOffset; ++i) {+ setFloat(i, mapWithIndexFunction.apply(getFloat(i), i));+ }+ return this;+ }++ private final FloatTensor addInPlace(final int thisOffset, final FloatTensor that, final int thatOffset, int size) {+ return mapWithIndexInPlace(thisOffset, size, (value, index) -> value + that.getFloat(index - thisOffset + thatOffset));+ }++ public final FloatTensor addInPlace(final FloatTensor that) {+ return addInPlace(0, that, 0, size());+ }++ private final FloatTensor multiplyInPlace(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {+ return mapWithIndexInPlace(thisOffset, size, (value, index) -> value * that.getFloat(index - thisOffset + thatOffset));+ }++ public final FloatTensor multiplyInPlace(final FloatTensor that) {+ return multiplyInPlace(0, that, 0, size());+ }++ public final FloatTensor divideInPlace(final int thisOffset, final int size, final float value) {+ return mapInPlace(thisOffset, size, f -> f / value);+ }++ public FloatTensor fillInPlace(final int thisOffset, final int size, final float value) {+ return mapInPlace(thisOffset, size, unused -> value);+ }++ public final FloatTensor softmaxInPlace(final int thisOffset, final int size) {+ // find max value (for numerical stability)+ float maxVal = max(thisOffset, size);+ // exp and sum+ mapInPlace(thisOffset, size, f -> (float) Math.exp(f - maxVal));+ float sum = sum(thisOffset, size);+ // normalize+ return divideInPlace(thisOffset, size, sum);+ }++ public FloatTensor saxpyInPlace(final int thisOffset, final FloatTensor that, final int thatOffset, final int size, final float a) {+ // this[thatOffset ... thatOffset + size) = a * that[thatOffset ... thatOffset + size) + this[thisOffset ... thisOffset + size)+ for (int i = 0; i < size; ++i) {+ this.setFloat(thisOffset + i, a * that.getFloat(thatOffset + i) + this.getFloat(thisOffset + i));+ }+ return this;+ }+}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Tensor/Q4_0FloatTensor.java b/source/net/yacy/ai/llama3/Tensor/Q4_0FloatTensor.javanew file mode 100644index 000000000..b4e60379a--- /dev/null+++ b/source/net/yacy/ai/llama3/Tensor/Q4_0FloatTensor.java@@ -0,0 +1,219 @@+/**+ * Q4_0FloatTensor.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Tensor;++import java.nio.ByteBuffer;+import java.lang.invoke.MethodHandles;+import java.lang.invoke.VarHandle;++import net.yacy.ai.llama3.Model.GGMLType;++public final class Q4_0FloatTensor extends FloatTensor {++ private final int size;+ private final ByteBuffer buffer;++ public Q4_0FloatTensor(final int size, final ByteBuffer buffer) {+ this.size = size;+ this.buffer = buffer;+ }++ private static final VarHandle FLOAT_ARRAY_HANDLE;++ static {+ try {+ FLOAT_ARRAY_HANDLE = MethodHandles.arrayElementVarHandle(float[].class);+ } catch (Exception e) {+ throw new RuntimeException("Failed to get VarHandle for float[]", e);+ }+ }++ @Override+ public final int size() {+ return size;+ }++ @Override+ public final void setFloat(final int index, final float value) {+ throw new UnsupportedOperationException("setFloat");+ }++ @Override+ public final GGMLType type() {+ return GGMLType.Q4_0;+ }++ private final static int LOG2_QUANT_BLOCK_SIZE = Integer.numberOfTrailingZeros(GGMLType.Q4_0.blockSize); // only if QUANT_BLOCK_SIZE == 2^LOG2_QUANT_BLOCK_SIZE+ private final static int QUANT_HALF_BLOCK = GGMLType.Q4_0.blockSize / 2; // 16+ private final static int QUANT_FLOAT16_BYTES = GGMLType.FLOAT16_BYTES; // 2++ @Override+ public final float getFloat(final int index) {+ assert 0 <= index && index < size;++ int blockIndex = index >>> LOG2_QUANT_BLOCK_SIZE; // index / QUANT_BLOCK_SIZE;+ int blockOffset = blockIndex * GGMLType.Q4_0.typeSize;+ final long offset = blockOffset;+ float scale = FloatTensor.float16ToFloat(buffer.getShort((int) offset));+ final int modIndex = index & (GGMLType.Q4_0.blockSize - 1); //index % QUANT_BLOCK_SIZE;+ final boolean isLow = modIndex < QUANT_HALF_BLOCK;+ final int adjustedIndex = modIndex - (isLow ? 0 : QUANT_HALF_BLOCK);+ final int dataIndex = blockOffset + QUANT_FLOAT16_BYTES + adjustedIndex;+ final int packed = (buffer.get((int) (long) dataIndex)) & 0xFF;+ final int nibble = isLow ? (packed & 0x0F) : ((packed >>> 4) & 0x0F);+ //final float quant = nibble - 8;+ return (nibble - 8) * scale;+ }++ public final void getFloatArray(final int index, final float[] out, final int outOffset, final int length) {+ int inPos = index;+ int outPos = outOffset;+ final int end = index + length;++ while (inPos < end) {+ final int blockIndex = inPos >>> LOG2_QUANT_BLOCK_SIZE;+ final int blockOffset = blockIndex * GGMLType.Q4_0.typeSize;+ float scale = FloatTensor.float16ToFloat(buffer.getShort((int) (long) blockOffset));++ final int blockStart = blockIndex * GGMLType.Q4_0.blockSize;+ final int blockEnd = Math.min(blockStart + GGMLType.Q4_0.blockSize, end);++ // Process two values at a time+ int i = inPos;+ while (i + 1 < blockEnd) {+ int blockMod0 = i & (GGMLType.Q4_0.blockSize - 1);+ int blockMod1 = blockMod0 + 1;++ int byteOffset0 = blockOffset + QUANT_FLOAT16_BYTES + (blockMod0 < QUANT_HALF_BLOCK ? blockMod0 : blockMod0 - QUANT_HALF_BLOCK);+ int byteOffset1 = blockOffset + QUANT_FLOAT16_BYTES + (blockMod1 < QUANT_HALF_BLOCK ? blockMod1 : blockMod1 - QUANT_HALF_BLOCK);+ final long offset = byteOffset0;++ int packed0 = buffer.get((int) offset) & 0xFF;+ final long offset1 = byteOffset1;+ int packed1 = (byteOffset1 == byteOffset0) ? packed0 : (buffer.get((int) offset1) & 0xFF);++ // Decode first value from packed0+ int nibble0 = (blockMod0 < QUANT_HALF_BLOCK) ? (packed0 & 0x0F) : ((packed0 >>> 4) & 0x0F);+ out[outPos++] = (nibble0 - 8) * scale;++ // Decode second value from packed1+ int nibble1 = (blockMod1 < QUANT_HALF_BLOCK) ? (packed1 & 0x0F) : ((packed1 >>> 4) & 0x0F);+ out[outPos++] = (nibble1 - 8) * scale;++ i += 2;+ }++ // Handle last element if length is odd+ if (i < blockEnd) {+ int blockMod = i & (GGMLType.Q4_0.blockSize - 1);+ int byteOffset = blockOffset + QUANT_FLOAT16_BYTES + (blockMod < QUANT_HALF_BLOCK ? blockMod : blockMod - QUANT_HALF_BLOCK);+ final long offset = byteOffset;+ int packed = buffer.get((int) offset) & 0xFF;+ int nibble = (blockMod < QUANT_HALF_BLOCK) ? (packed & 0x0F) : ((packed >>> 4) & 0x0F);+ out[outPos++] = (nibble - 8) * scale;+ i++;+ }++ int copied = blockEnd - inPos;+ inPos += copied;+ }+ }++ public static final ThreadLocal<float[]> scratchBuffer = ThreadLocal.withInitial(() -> new float[GGMLType.Q4_0.blockSize]);++ @Override+ public void copyTo(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {+ final float[] decoded = scratchBuffer.get();+ int remaining = size;+ int srcIndex = thisOffset;+ int dstIndex = thatOffset;++ // Decode and copy in QUANT_BLOCK_SIZE chunks+ while (remaining >= GGMLType.Q4_0.blockSize) {+ getFloatArray(srcIndex, decoded, 0, GGMLType.Q4_0.blockSize);+ for (int i = 0; i < GGMLType.Q4_0.blockSize; i++) {+ that.setFloat(dstIndex + i, (float) FLOAT_ARRAY_HANDLE.get(decoded, i));+ }+ srcIndex += GGMLType.Q4_0.blockSize;+ dstIndex += GGMLType.Q4_0.blockSize;+ remaining -= GGMLType.Q4_0.blockSize;+ }++ // Decode and copy any leftover elements one by one+ if (remaining > 0) {+ getFloatArray(srcIndex, decoded, 0, remaining);+ for (int i = 0; i < remaining; i++) {+ that.setFloat(dstIndex + i, (float) FLOAT_ARRAY_HANDLE.get(decoded, i));+ }+ }+ }++ /**+ * dot product which has the getFloat method inlined in such a way that it processes full blocks at once.+ * This gains a > 2.5 times token/s performance increase compared to the generic dot-getFloat implementation.+ */+ public final float dot(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {+ float result = 0.0f;+ int index = 0;+ final int blockLimit = size - (size % GGMLType.Q4_0.blockSize);++ // Process full blocks, one block are 32 elements, 16 quantized values and 1 scale+ while (index < blockLimit) {++ // Get this block+ final int thisBlockIndex = (thisOffset + index) >>> LOG2_QUANT_BLOCK_SIZE; // (thisOffset + index) / QUANT_BLOCK_SIZE;+ final int thisBlockOffset = thisBlockIndex * GGMLType.Q4_0.typeSize;+ final float thisScale = FloatTensor.float16ToFloat(buffer.getShort((int) (long) thisBlockOffset));++ // Process block: read all quantized values from this block at once+ final int quantOffset = thisBlockOffset + QUANT_FLOAT16_BYTES;+ float blockResult = 0.0f;+ final int thatIndex = thatOffset + index;+ if (that instanceof ArrayFloatTensor) {+ final ArrayFloatTensor thatArray = (ArrayFloatTensor) that;+ final float[] b = thatArray.values;+ for (int i = 0; i < QUANT_HALF_BLOCK; ++i) {+ final byte packed = buffer.get(quantOffset + i);+ final float valB0 = (float) FLOAT_ARRAY_HANDLE.get(b, thatIndex + i);+ final float valB1 = (float) FLOAT_ARRAY_HANDLE.get(b, thatIndex + i + QUANT_HALF_BLOCK);+ blockResult += ((packed & 0x0F) - 8) * valB0 + (((packed >>> 4) & 0x0F) - 8) * valB1;+ }+ } else {+ for (int i = 0; i < QUANT_HALF_BLOCK; ++i) {+ final byte packed = buffer.get(quantOffset + i);+ blockResult += ((packed & 0x0F) - 8) * that.getFloat(thatIndex + i) + (((packed >>> 4) & 0x0F) - 8) * that.getFloat(thatIndex + i + QUANT_HALF_BLOCK);+ }+ }+ result += blockResult * thisScale;+ index += GGMLType.Q4_0.blockSize;+ }++ // Process remaining elements+ for (; index < size; index++) {+ result += this.getFloat(thisOffset + index) * that.getFloat(thatOffset + index);+ }++ return (float) result;+ }++}\ No newline at end of filediff --git a/source/net/yacy/ai/llama3/Tensor/Q8_0FloatTensor.java b/source/net/yacy/ai/llama3/Tensor/Q8_0FloatTensor.javanew file mode 100644index 000000000..5e84b86fb--- /dev/null+++ b/source/net/yacy/ai/llama3/Tensor/Q8_0FloatTensor.java@@ -0,0 +1,134 @@+/**+ * Q8_0FloatTensor.java++ * This file was extracted from the llama3/qwen2 projects+ * https://github.com/mukel/llama3.java+ * https://github.com/mukel/qwen2.svm.java+ *+ * License: MIT License+ *+ * Copyright (c) 2024 Andrej Karpathy (for llama2.c)+ * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2)+ * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp)+ * Copyright (c) 2025 Michael Peter Christen for modifications:+ * The code was modified to fit the YaCy AI project:+ * - back-port to Java 11 (removal of Vector API operations and record types)+ * - removal of interactive mode and system.out printing+ * - separation of the classes in the single java and refactoring+ * - run-time performance optimizations for dot product computation of quantized values+ * - joining of llama3/qwen2 into one code base; multi-arch options+ * - alignment with code from https://github.com/ggml-org/llama.cpp/+ */++package net.yacy.ai.llama3.Tensor;++import java.nio.ByteBuffer;++import net.yacy.ai.llama3.Model.GGMLType;++public final class Q8_0FloatTensor extends FloatTensor {++ final int size;+ final ByteBuffer buffer;+++ public Q8_0FloatTensor(final int size, final ByteBuffer buffer) {+ this.size = size;+ this.buffer = buffer;+ }++ @Override+ public int size() {+ return size;+ }++ @Override+ public void setFloat(final int index, final float value) {+ throw new UnsupportedOperationException("setFloat");+ }++ @Override+ public GGMLType type() {+ return GGMLType.Q8_0;+ }++ @Override+ public final float getFloat(final int index) {+ assert 0 <= index && index < size;+ int blockIndex = index / GGMLType.Q8_0.blockSize;+ int withinBlockIndex = index % GGMLType.Q8_0.blockSize;+ int blockOffset = blockIndex * GGMLType.Q8_0.typeSize;+ byte quant = buffer.get((int) (long) (blockOffset + GGMLType.FLOAT16_BYTES + withinBlockIndex));+ final long offset = blockOffset;+ float scale = FloatTensor.float16ToFloat(buffer.getShort((int) offset));+ return quant * scale;+ }++ @Override+ public float dot(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {+ assert 0 <= thisOffset && thisOffset + size <= this.size;+ assert 0 <= thatOffset && thatOffset + size <= that.size();++ float result = 0f;++ // Calculate first and last block indices+ int firstBlock = thisOffset / GGMLType.Q8_0.blockSize;+ int lastBlock = (thisOffset + size - 1) / GGMLType.Q8_0.blockSize;++ for (int block = firstBlock; block <= lastBlock; block++) {+ // Calculate block boundaries and overlaps+ int blockStart = block * GGMLType.Q8_0.blockSize;+ int blockEnd = blockStart + GGMLType.Q8_0.blockSize;+ int start = Math.max(thisOffset, blockStart);+ int end = Math.min(thisOffset + size, blockEnd);+ int length = end - start;++ // Get common scale factor for this block+ int blockOffset = block * GGMLType.Q8_0.typeSize;+ final long offset = blockOffset;+ float thisScale = FloatTensor.float16ToFloat(buffer.getShort((int) offset));++ // Compute sum of products for this block+ float blockSum = 0f;+ int withinBlockStart = start % GGMLType.Q8_0.blockSize;++ int memOffset = blockOffset + GGMLType.FLOAT16_BYTES + withinBlockStart;+ int thatoffset = thatOffset + (start - thisOffset);+ if (that instanceof ArrayFloatTensor) {+ ArrayFloatTensor thatArray = (ArrayFloatTensor) that;+ for (int i = 0; i < length; i++) {+ blockSum += buffer.get((int) (long) memOffset++) * thatArray.getFloat(thatoffset++);+ }+ } else {+ for (int i = 0; i < length; i++) {+ blockSum += buffer.get((int) (long) memOffset++) * that.getFloat(thatoffset++);+ }+ }++ // Apply scale to the entire block sum+ result += thisScale * blockSum;+ }++ return result;+ }++ @Override+ public void copyTo(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {+ assert 0 <= thisOffset && thisOffset + size <= this.size;+ assert 0 <= thatOffset && thatOffset + size <= that.size();++ final int endOffset = thatOffset + size;++ for (int i = thatOffset; i < endOffset; ++i) {+ int index = i - thatOffset + thisOffset;+ int blockIndex = index / GGMLType.Q8_0.blockSize;+ int withinBlockIndex = index % GGMLType.Q8_0.blockSize;+ int blockOffset = blockIndex * GGMLType.Q8_0.typeSize;++ byte quant = buffer.get((int) (long) (blockOffset + GGMLType.FLOAT16_BYTES + withinBlockIndex));+ final long offset = blockOffset;+ float scale = FloatTensor.float16ToFloat(buffer.getShort((int) offset));+ that.setFloat(i, quant * scale);+ }+ }+}