Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
refactoring of llm classes
Michael Peter Christen1ab5ee347649
diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.javanew file mode 100644index 000000000..760b975e2--- /dev/null+++ b/source/net/yacy/ai/LLM.java@@ -0,0 +1,339 @@+/**+ * LLMEndpoint+ * Copyright 2024 by Michael Peter Christen+ * First released 17.05.2024 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai;++import java.io.BufferedReader;+import java.io.IOException;+import java.io.InputStreamReader;+import java.io.OutputStream;+import java.net.HttpURLConnection;+import java.net.URI;+import java.net.URISyntaxException;+import java.net.URL;+import java.util.AbstractMap;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++public class LLM {++ private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"};++ public static enum LLMType {+ OPENAI("https://api.openai.com"),+ OLLAMA("http://localhost:11434"),+ LMSTUDIO("http://localhost:1234"),+ OPENROUTER("https://openrouter.ai/api"),+ OTHER(null);+ public String hoststub;+ private LLMType(String hoststub) {+ this.hoststub = hoststub;+ }+ }++ public final String hoststub;+ public final String key; // the apikey+ public final int max_tokens; // the max_tokens as configured by the endpoint for all models+ public final LLMType type;++ public LLM(final String hoststub, String key, final int max_tokens, final LLMType type) {+ this.hoststub = hoststub.endsWith("/") ? hoststub.substring(0, hoststub.length() - 1) : hoststub;+ this.key = key == null ? "" : key;+ this.max_tokens = max_tokens <= 0 ? 4096 : max_tokens;+ this.type = type;+ }++ public String getHoststub() {+ return this.hoststub;+ }+++ // API Helper Methods++ public static String sendPostRequest(final String urls, final JSONObject data) throws IOException, URISyntaxException {+ final URL url = new URI(urls).toURL();+ final HttpURLConnection conn = (HttpURLConnection) url.openConnection();+ conn.setRequestMethod("POST");+ conn.setRequestProperty("Content-Type", "application/json");+ conn.setDoOutput(true);++ try (OutputStream os = conn.getOutputStream()) {+ final byte[] input = data.toString().getBytes("utf-8");+ os.write(input, 0, input.length);+ }++ final int responseCode = conn.getResponseCode();+ if (responseCode == HttpURLConnection.HTTP_OK) {+ try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {+ final StringBuilder response = new StringBuilder();+ String responseLine;+ while ((responseLine = br.readLine()) != null) {+ response.append(responseLine.trim());+ }+ return response.toString();+ }+ } else {+ throw new IOException("Request failed with response code " + responseCode);+ }+ }++ public static String sendGetRequest(final String urls) throws IOException, URISyntaxException {+ final URL url = new URI(urls).toURL();+ final HttpURLConnection conn = (HttpURLConnection) url.openConnection();+ conn.setRequestMethod("GET");++ final int responseCode = conn.getResponseCode();+ if (responseCode == HttpURLConnection.HTTP_OK) {+ try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {+ final StringBuilder response = new StringBuilder();+ String responseLine;+ while ((responseLine = br.readLine()) != null) {+ response.append(responseLine.trim());+ }+ return response.toString();+ }+ } else {+ throw new IOException("Request failed with response code " + responseCode);+ }+ }+++ public LinkedHashMap<String, Long> listOllamaModels() {+ final LinkedHashMap<String, Long> sortedMap = new LinkedHashMap<>();+ try {+ final String response = sendGetRequest(this.hoststub + "/api/tags");+ final JSONObject responseObject = new JSONObject(response);+ final JSONArray models = responseObject.getJSONArray("models");++ final List<Map.Entry<String, Long>> list = new ArrayList<>();+ for (int i = 0; i < models.length(); i++) {+ final JSONObject model = models.getJSONObject(i);+ final String name = model.optString("name", "");+ final long size = model.optLong("size", 0);+ list.add(new AbstractMap.SimpleEntry<>(name, size));+ }++ // Sort the list in descending order based on the values+ list.sort((o1, o2) -> o2.getValue().compareTo(o1.getValue()));++ // Create a new LinkedHashMap and add the sorted entries+ for (final Map.Entry<String, Long> entry : list) {+ sortedMap.put(entry.getKey(), entry.getValue());+ }+ } catch (JSONException | URISyntaxException | IOException e) {+ e.printStackTrace();+ }+ return sortedMap;+ }++ public boolean ollamaModelExists(final String name) {+ final JSONObject data = new JSONObject();+ try {+ data.put("name", name);+ sendPostRequest(this.hoststub + "/api/show", data);+ return true;+ } catch (JSONException | URISyntaxException | IOException e) {+ return false;+ }+ }++ public boolean pullOllamaModel(final String name) {+ final JSONObject data = new JSONObject();+ try {+ data.put("name", name);+ data.put("stream", false);+ final String response = sendPostRequest(this.hoststub + "/api/pull", data);+ // this sends {"status": "success"} in case of success+ final JSONObject responseObject = new JSONObject(response);+ final String status = responseObject.optString("status", "");+ return status.equals("success");+ } catch (JSONException | URISyntaxException | IOException e) {+ return false;+ }+ }++ // chat endpoints++ public static class Context extends JSONArray {+ public Context(String systemPrompt) throws JSONException {+ super();+ final JSONObject systemPromptObject = new JSONObject(true);+ systemPromptObject.put("role", "system");+ systemPromptObject.put("content", systemPrompt);+ this.put(systemPromptObject);+ }+ public void addDialog(String user, String assistant) throws JSONException {+ final JSONObject userPromptObject = new JSONObject(true);+ userPromptObject.put("role", "user");+ userPromptObject.put("content", user);+ this.put(userPromptObject);+ final JSONObject assistantPromptObject = new JSONObject(true);+ assistantPromptObject.put("role", "assistant");+ assistantPromptObject.put("content", assistant);+ this.put(assistantPromptObject);+ }+ public void addPrompt(String userPrompt) throws JSONException {+ final JSONObject userPromptObject = new JSONObject(true);+ userPromptObject.put("role", "user");+ userPromptObject.put("content", userPrompt);+ this.put(userPromptObject);+ }+ }++ // OpenAI chat client, works with llama.cpp and Ollama+ public String chat(final String model, final Context context, JSONObject schema, final int max_tokens) throws IOException {+ final JSONObject data = new JSONObject();++ try {+ data.put("model", model);+ data.put("temperature", 0.1);+ data.put("max_tokens", max_tokens);+ data.put("messages", context);+ data.put("stop", new JSONArray(STOPTOKENS));+ data.put("stream", false);++ if (schema != null) {+ System.out.println(schema.toString());+ JSONObject json_schema = new JSONObject(true);+ json_schema.put("strict", true);+ json_schema.put("schema", schema);+ JSONObject response_format = new JSONObject();+ response_format.put("type", "json_schema");+ response_format.put("json_schema", json_schema);+ data.put("response_format", response_format);+ }++ final String response = sendPostRequest(this.hoststub + "/v1/chat/completions", data);+ final JSONObject responseObject = new JSONObject(response);+ final JSONArray choices = responseObject.getJSONArray("choices");+ final JSONObject choice = choices.getJSONObject(0);+ final JSONObject message = choice.getJSONObject("message");+ final String content = message.optString("content", "");+ return content;+ } catch (JSONException | URISyntaxException e) {+ throw new IOException(e.getMessage());+ }+ }++ public String chat(final String model, final String systemPrompt, final String userPrompt, final int max_tokens) throws IOException {+ try {+ Context context = new Context(systemPrompt);+ context.addPrompt(userPrompt);+ return chat(model, context, null, max_tokens);+ } catch (JSONException e) {+ throw new IOException(e.getMessage());+ }+ }++ public static String[] stringsFromChat(String chatanswer) throws JSONException {+ JSONArray ja = new JSONArray(chatanswer);+ List<String> list = new ArrayList<>();+ // parse the JSON array and extract strings+ for (int i = 0; i < ja.length(); i++) {+ Object item = ja.get(i);+ if (item instanceof String) {+ list.add((String) item);+ } else if (item instanceof JSONObject) {+ JSONObject jo = (JSONObject) item;+ String answer = jo.optString("answer", null);+ if (answer != null) {+ list.add(answer);+ } else {+ // take any string value from the object+ for (String key : jo.keySet()) {+ Object value = jo.optString(key, null);+ if (value != null && value instanceof String) {+ list.add((String) value);+ break; // take the first string found+ }+ }+ }+ }+ }+ // convert the list to an array+ String[] result = new String[list.size()];+ return list.toArray(result);+ }++ public final static JSONObject listSchema = new JSONObject(Map.of(+ "title", "Answer List",+ "type", "array",+ "properties", Map.of(+ "answer", Map.of("type", "string")+ ),+ "required", List.of("answer")+ ));++ public static void main(final String[] args) {+ final LLM llm = new LLM(LLMType.OLLAMA.hoststub, null, 4069, LLMType.OLLAMA);++ final LinkedHashMap<String, Long> models = llm.listOllamaModels();+ System.out.println(models.toString());++ // check if model exists+ final String model = "qwen2.5:0.5b";+ if (llm.ollamaModelExists(model))+ System.out.println("model " + model + " exists");+ else+ System.out.println("model " + model + " does not exist");++ // pull a model+ final boolean success = llm.pullOllamaModel(model);+ System.out.println("pulled model: " + model + ": " + success);++ String response;+ try {+ response = llm.chat(model, "You are a helpful assistant.", "What is the capital of France?", 1000);+ System.out.println("Chat response: " + response);+ } catch (IOException e) {++ e.printStackTrace();+ }++ // make chat completion with model+ String question = "Who invented the wheel?";+ try {+ final String answer = llm.chat(model, "Make short answers.", question, 200);+ System.out.println(answer);+ } catch (final IOException e) {+ e.printStackTrace();+ }++ // try the json parser from chat results+ question = "Make a list of four names from Star Wars movies. Use a JSON Array.";+ try {+ Context context = new Context("Make short answers");+ context.addPrompt(question);+ final String[] a = stringsFromChat(llm.chat(model, context, listSchema, 1000));+ for (String s : a) {+ System.out.println(s);+ }+ } catch (final IOException | JSONException e) {+ e.printStackTrace();+ }+ }++}diff --git a/source/net/yacy/ai/OllamaSwarm.java b/source/net/yacy/ai/LLMSwarm.javasimilarity index 83%rename from source/net/yacy/ai/OllamaSwarm.javarename to source/net/yacy/ai/LLMSwarm.javaindex 9206f13dd..9aa3286b2 100644--- a/source/net/yacy/ai/OllamaSwarm.java+++ b/source/net/yacy/ai/LLMSwarm.java@@ -28,26 +28,22 @@ import java.util.Map;import java.util.Random;-public class OllamaSwarm {+public class LLMSwarm {Random random = new Random();- List<OllamaClient> swarm;;+ List<LLM> swarm;;- public OllamaSwarm() {+ public LLMSwarm() {this.swarm = new ArrayList<>();}- public void addOllamaClient(final OllamaClient client) {+ public void addOllamaClient(final LLM client) {this.swarm.add(client);}- public void addOllamaClient(final String hoststub) {- this.swarm.add(new OllamaClient(hoststub));- }-public List<String> getSwarm() {List<String> hoststubs = new ArrayList<>();- for (OllamaClient client : this.swarm) {+ for (LLM client : this.swarm) {hoststubs.add(client.getHoststub());}return hoststubs;@@ -59,7 +55,7 @@ public class OllamaSwarm {*/public LinkedHashMap<String, Integer> listOllamaModels() {final LinkedHashMap<String, Integer> modelCountMap = new LinkedHashMap<>();- for (OllamaClient client: this.swarm) {+ for (LLM client: this.swarm) {LinkedHashMap<String, Long> models = client.listOllamaModels();for (Map.Entry<String, Long> entry : models.entrySet()) {modelCountMap.merge(entry.getKey(), 1, Integer::sum);@@ -74,7 +70,7 @@ public class OllamaSwarm {* @return true if at least one client has the model, false otherwise*/public boolean ollamaModelExists(final String name) {- for (OllamaClient client : this.swarm) {+ for (LLM client : this.swarm) {if (client.ollamaModelExists(name)) {return true;}@@ -91,7 +87,7 @@ public class OllamaSwarm {public boolean pullOllamaModel(final String name, boolean all) {boolean pulled = false;if (all) {- for (OllamaClient client: this.swarm) {+ for (LLM client: this.swarm) {if (client.pullOllamaModel(name)) {pulled = true;}@@ -100,7 +96,7 @@ public class OllamaSwarm {// we try several times in case one client is not availablefor (int i = 0; i < this.swarm.size(); i++) {int index = random.nextInt(this.swarm.size());- OllamaClient client = this.swarm.get(index);+ LLM client = this.swarm.get(index);pulled = client.pullOllamaModel(name);if (pulled) return true;}@@ -113,9 +109,9 @@ public class OllamaSwarm {* @param name* @return a List of OllamaClient objects that have the specified model*/- public List<OllamaClient> getSwarm(String name) {- List<OllamaClient> clientsWithModel = new ArrayList<>();- for (OllamaClient client : this.swarm) {+ public List<LLM> getSwarm(String name) {+ List<LLM> clientsWithModel = new ArrayList<>();+ for (LLM client : this.swarm) {if (client.ollamaModelExists(name)) {clientsWithModel.add(client);}@@ -132,11 +128,11 @@ public class OllamaSwarm {* @throws IOException*/public String getAnswer(String model, String systemPrompt, String userPrompt) throws IOException {- List<OllamaClient> clientsWithModel = getSwarm(model);+ List<LLM> clientsWithModel = getSwarm(model);if (clientsWithModel.isEmpty()) {return "No client with model " + model + " found.";}- OllamaClient client = clientsWithModel.get(random.nextInt(clientsWithModel.size()));+ LLM client = clientsWithModel.get(random.nextInt(clientsWithModel.size()));return client.chat(model, systemPrompt, userPrompt, 4096);}diff --git a/source/net/yacy/ai/OllamaClient.java b/source/net/yacy/ai/OllamaClient.javadeleted file mode 100644index 53aa66742..000000000--- a/source/net/yacy/ai/OllamaClient.java+++ /dev/null@@ -1,129 +0,0 @@-/**- * OllamaClient- * Copyright 2024 by Michael Peter Christen- * First released 17.05.2024 at https://yacy.net- *- * This library is free software; you can redistribute it and/or- * modify it under the terms of the GNU Lesser General Public- * License as published by the Free Software Foundation; either- * version 2.1 of the License, or (at your option) any later version.- *- * This library is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU- * Lesser General Public License for more details.- *- * You should have received a copy of the GNU Lesser General Public License- * along with this program in the file lgpl21.txt- * If not, see <http://www.gnu.org/licenses/>.- */--package net.yacy.ai;--import java.io.IOException;-import java.net.URISyntaxException;-import java.util.AbstractMap;-import java.util.ArrayList;-import java.util.LinkedHashMap;-import java.util.List;-import java.util.Map;--import org.json.JSONArray;-import org.json.JSONException;-import org.json.JSONObject;--public class OllamaClient extends OpenAIClient {-- public static String OLLAMA_API_HOST = "http://localhost:11434";-- public OllamaClient(final String hoststub) {- super(hoststub);- }-- public String getHoststub() {- return this.hoststub;- }-- public LinkedHashMap<String, Long> listOllamaModels() {- final LinkedHashMap<String, Long> sortedMap = new LinkedHashMap<>();- try {- final String response = OpenAIClient.sendGetRequest(this.hoststub + "/api/tags");- final JSONObject responseObject = new JSONObject(response);- final JSONArray models = responseObject.getJSONArray("models");-- final List<Map.Entry<String, Long>> list = new ArrayList<>();- for (int i = 0; i < models.length(); i++) {- final JSONObject model = models.getJSONObject(i);- final String name = model.optString("name", "");- final long size = model.optLong("size", 0);- list.add(new AbstractMap.SimpleEntry<>(name, size));- }-- // Sort the list in descending order based on the values- list.sort((o1, o2) -> o2.getValue().compareTo(o1.getValue()));-- // Create a new LinkedHashMap and add the sorted entries- for (final Map.Entry<String, Long> entry : list) {- sortedMap.put(entry.getKey(), entry.getValue());- }- } catch (JSONException | URISyntaxException | IOException e) {- e.printStackTrace();- }- return sortedMap;- }-- public boolean ollamaModelExists(final String name) {- final JSONObject data = new JSONObject();- try {- data.put("name", name);- OpenAIClient.sendPostRequest(this.hoststub + "/api/show", data);- return true;- } catch (JSONException | URISyntaxException | IOException e) {- return false;- }- }-- public boolean pullOllamaModel(final String name) {- final JSONObject data = new JSONObject();- try {- data.put("name", name);- data.put("stream", false);- final String response = OpenAIClient.sendPostRequest(this.hoststub + "/api/pull", data);- // this sends {"status": "success"} in case of success- final JSONObject responseObject = new JSONObject(response);- final String status = responseObject.optString("status", "");- return status.equals("success");- } catch (JSONException | URISyntaxException | IOException e) {- return false;- }- }-- public static void main(final String[] args) {- final OllamaClient oc = new OllamaClient(OLLAMA_API_HOST);-- final LinkedHashMap<String, Long> models = oc.listOllamaModels();- System.out.println(models.toString());-- // check if model exists- final String model = "qwen2.5:0.5b";- if (oc.ollamaModelExists(model))- System.out.println("model " + model + " exists");- else- System.out.println("model " + model + " does not exist");-- // pull a model- final boolean success = oc.pullOllamaModel(model);- System.out.println("pulled model: " + model + ": " + success);-- String response;- try {- response = oc.chat(model, "You are a helpful assistant.", "What is the capital of France?", 1000);- System.out.println("Chat response: " + response);- } catch (IOException e) {-- e.printStackTrace();- }-- }--}diff --git a/source/net/yacy/ai/OpenAIClient.java b/source/net/yacy/ai/OpenAIClient.javadeleted file mode 100644index cee112495..000000000--- a/source/net/yacy/ai/OpenAIClient.java+++ /dev/null@@ -1,237 +0,0 @@-/**- * OpenAIClient- * Copyright 2024 by Michael Peter Christen- * First released 17.05.2024 at https://yacy.net- *- * This library is free software; you can redistribute it and/or- * modify it under the terms of the GNU Lesser General Public- * License as published by the Free Software Foundation; either- * version 2.1 of the License, or (at your option) any later version.- *- * This library is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU- * Lesser General Public License for more details.- *- * You should have received a copy of the GNU Lesser General Public License- * along with this program in the file lgpl21.txt- * If not, see <http://www.gnu.org/licenses/>.- */--package net.yacy.ai;--import java.io.BufferedReader;-import java.io.IOException;-import java.io.InputStreamReader;-import java.io.OutputStream;-import java.net.HttpURLConnection;-import java.net.URI;-import java.net.URISyntaxException;-import java.net.URL;-import java.util.ArrayList;-import java.util.List;-import java.util.Map;--import org.json.JSONArray;-import org.json.JSONException;-import org.json.JSONObject;---public class OpenAIClient {-- private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"};-- protected final String hoststub;-- public OpenAIClient(final String hoststub) {- this.hoststub = hoststub;- }--- // API Helper Methods-- public static String sendPostRequest(final String endpoint, final JSONObject data) throws IOException, URISyntaxException {- final URL url = new URI(endpoint).toURL();- final HttpURLConnection conn = (HttpURLConnection) url.openConnection();- conn.setRequestMethod("POST");- conn.setRequestProperty("Content-Type", "application/json");- conn.setDoOutput(true);-- try (OutputStream os = conn.getOutputStream()) {- final byte[] input = data.toString().getBytes("utf-8");- os.write(input, 0, input.length);- }-- final int responseCode = conn.getResponseCode();- if (responseCode == HttpURLConnection.HTTP_OK) {- try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {- final StringBuilder response = new StringBuilder();- String responseLine;- while ((responseLine = br.readLine()) != null) {- response.append(responseLine.trim());- }- return response.toString();- }- } else {- throw new IOException("Request failed with response code " + responseCode);- }- }-- public static String sendGetRequest(final String endpoint) throws IOException, URISyntaxException {- final URL url = new URI(endpoint).toURL();- final HttpURLConnection conn = (HttpURLConnection) url.openConnection();- conn.setRequestMethod("GET");-- final int responseCode = conn.getResponseCode();- if (responseCode == HttpURLConnection.HTTP_OK) {- try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {- final StringBuilder response = new StringBuilder();- String responseLine;- while ((responseLine = br.readLine()) != null) {- response.append(responseLine.trim());- }- return response.toString();- }- } else {- throw new IOException("Request failed with response code " + responseCode);- }- }-- public static class Context extends JSONArray {- public Context(String systemPrompt) throws JSONException {- super();- final JSONObject systemPromptObject = new JSONObject(true);- systemPromptObject.put("role", "system");- systemPromptObject.put("content", systemPrompt);- this.put(systemPromptObject);- }- public void addDialog(String user, String assistant) throws JSONException {- final JSONObject userPromptObject = new JSONObject(true);- userPromptObject.put("role", "user");- userPromptObject.put("content", user);- this.put(userPromptObject);- final JSONObject assistantPromptObject = new JSONObject(true);- assistantPromptObject.put("role", "assistant");- assistantPromptObject.put("content", assistant);- this.put(assistantPromptObject);- }- public void addPrompt(String userPrompt) throws JSONException {- final JSONObject userPromptObject = new JSONObject(true);- userPromptObject.put("role", "user");- userPromptObject.put("content", userPrompt);- this.put(userPromptObject);- }- }-- // OpenAI chat client, works with llama.cpp and Ollama-- public String chat(final String model, final Context context, JSONObject schema, final int max_tokens) throws IOException {- final JSONObject data = new JSONObject();-- try {- data.put("model", model);- data.put("temperature", 0.1);- data.put("max_tokens", max_tokens);- data.put("messages", context);- data.put("stop", new JSONArray(STOPTOKENS));- data.put("stream", false);-- if (schema != null) {- System.out.println(schema.toString());- JSONObject json_schema = new JSONObject(true);- json_schema.put("strict", true);- json_schema.put("schema", schema);- JSONObject response_format = new JSONObject();- response_format.put("type", "json_schema");- response_format.put("json_schema", json_schema);- data.put("response_format", response_format);- }-- final String response = sendPostRequest(this.hoststub + "/v1/chat/completions", data);- final JSONObject responseObject = new JSONObject(response);- final JSONArray choices = responseObject.getJSONArray("choices");- final JSONObject choice = choices.getJSONObject(0);- final JSONObject message = choice.getJSONObject("message");- final String content = message.optString("content", "");- return content;- } catch (JSONException | URISyntaxException e) {- throw new IOException(e.getMessage());- }- }-- public String chat(final String model, final String systemPrompt, final String userPrompt, final int max_tokens) throws IOException {- try {- Context context = new Context(systemPrompt);- context.addPrompt(userPrompt);- return chat(model, context, null, max_tokens);- } catch (JSONException e) {- throw new IOException(e.getMessage());- }- }-- public static String[] stringsFromChat(String chatanswer) throws JSONException {- JSONArray ja = new JSONArray(chatanswer);- List<String> list = new ArrayList<>();- // parse the JSON array and extract strings- for (int i = 0; i < ja.length(); i++) {- Object item = ja.get(i);- if (item instanceof String) {- list.add((String) item);- } else if (item instanceof JSONObject) {- JSONObject jo = (JSONObject) item;- String answer = jo.optString("answer", null);- if (answer != null) {- list.add(answer);- } else {- // take any string value from the object- for (String key : jo.keySet()) {- Object value = jo.optString(key, null);- if (value != null && value instanceof String) {- list.add((String) value);- break; // take the first string found- }- }- }- }- }- // convert the list to an array- String[] result = new String[list.size()];- return list.toArray(result);- }-- public final static JSONObject listSchema = new JSONObject(Map.of(- "title", "Answer List",- "type", "array",- "properties", Map.of(- "answer", Map.of("type", "string")- ),- "required", List.of("answer")- ));-- public static void main(final String[] args) {- final String model = "qwen2.5:0.5b";- final OpenAIClient oaic = new OpenAIClient(OllamaClient.OLLAMA_API_HOST);- // make chat completion with model- String question = "Who invented the wheel?";- try {- final String answer = oaic.chat(model, "Make short answers.", question, 200);- System.out.println(answer);- } catch (final IOException e) {- e.printStackTrace();- }-- // try the json parser from chat results- question = "Make a list of four names from Star Wars movies. Use a JSON Array.";- try {- Context context = new Context("Make short answers");- context.addPrompt(question);- final String[] a = stringsFromChat(oaic.chat(model, context, listSchema, 1000));- for (String s : a) {- System.out.println(s);- }- } catch (final IOException | JSONException e) {- e.printStackTrace();- }- }--}diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.javaindex bbc891c75..63c28c284 100644--- a/source/net/yacy/http/servlets/RAGProxyServlet.java+++ b/source/net/yacy/http/servlets/RAGProxyServlet.java@@ -48,7 +48,7 @@ import org.json.JSONArray;import org.json.JSONException;import org.json.JSONObject;-import net.yacy.ai.OpenAIClient;+import net.yacy.ai.LLM;import net.yacy.cora.federate.solr.SolrType;import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;import net.yacy.search.Switchboard;@@ -257,10 +257,10 @@ public class RAGProxyServlet extends HttpServlet {StringBuilder query = new StringBuilder();String question = "Make a list of a maximum of four search words for the following question; use a JSON Array: " + prompt;try {- OpenAIClient oaic = new OpenAIClient(LLM_API_HOST);- OpenAIClient.Context context = new OpenAIClient.Context(LLM_SYSTEM_PREFIX);+ LLM llm = new LLM(LLM_API_HOST, null, 4096, LLM.LLMType.OLLAMA);+ LLM.Context context = new LLM.Context(LLM_SYSTEM_PREFIX);context.addPrompt(question);- String[] a = OpenAIClient.stringsFromChat(oaic.chat(model, context, OpenAIClient.listSchema, 80));+ String[] a = LLM.stringsFromChat(llm.chat(model, context, LLM.listSchema, 80));for (String s : a)query.append(s).append(' ');return query.toString().trim();