Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
added persistence to production models table
Michael Peter Christen37c26bddf399
diff --git a/defaults/yacy.init b/defaults/yacy.initindex 63db597c0..d61ddafe8 100644--- a/defaults/yacy.init+++ b/defaults/yacy.init@@ -1422,3 +1422,5 @@ decoration.grafics.linkstructure = truedecoration.hostanalysis = falsedecoration.simpleheadernavbar = navbar-default+# ai settings+ai.production_models = {}\ No newline at end of filediff --git a/htroot/LLMSelection_p.html b/htroot/LLMSelection_p.htmlindex ff95e98d3..4d3362a44 100644--- a/htroot/LLMSelection_p.html+++ b/htroot/LLMSelection_p.html@@ -23,18 +23,18 @@const PRODUCTION_MODEL_FEATURE_COLUMN_END = 11;const PRODUCTION_MODEL_ACTION_COLUMN_INDEX = PRODUCTION_MODEL_TOTAL_COLUMNS - 1;const PRODUCTION_MODEL_COLUMN_NAMES = [- "Provider",- "Model",+ "service",+ "model","hoststub",- "API Key",- "Max Token",- "Answers",- "Chat",- "Translation",- "Q/A-Generation",- "Classification",- "Shortener",- "Vision"+ "api_key",+ "max_tokens",+ "answers",+ "chat",+ "translation",+ "qa-generation",+ "classification",+ "tldr-shortener",+ "vision"];const PRODUCTION_MODEL_SUBMIT_URL = "LLMSelection_p.html";@@ -121,20 +121,20 @@return fetchJsonOrThrow(`${hoststub}/v1/models`);}- async function requestModelsForProvider(provider, hoststub) {- return provider === "OLLAMA"+ async function requestModelsForService(service, hoststub) {+ return service === "OLLAMA"? fetchOllamaModels(hoststub): fetchOpenAICompatibleModels(hoststub);}- function handleModelLoadError(provider, error) {+ function handleModelLoadError(service, error) {console.error("Error fetching models:", error);const status = error && typeof error.status === "number" ? error.status : null;if (status) {const apikeyEl = document.getElementById("apikey");apiKeyValue = apikeyEl ? apikeyEl.value.trim() : "";- if (provider !== "OLLAMA" && provider !== "LMSTUDIO" && !apiKeyValue) {- alert("an api key is required for this provider");+ if (service !== "OLLAMA" && service !== "LMSTUDIO" && !apiKeyValue) {+ alert("an api_key is required for this service");} else {alert(`Failed to load models. HTTP status: ${status}`);}@@ -145,13 +145,13 @@async function loadModelList() {availableModels = [];- const provider = document.getElementById("provider").value;+ const service = document.getElementById("service").value;const hoststub = document.getElementById("hoststub").value;try {- const responsej = await requestModelsForProvider(provider, hoststub);- renderAvailableModels(provider, responsej);- if (provider === "OLLAMA") {+ const responsej = await requestModelsForService(service, hoststub);+ renderAvailableModels(service, responsej);+ if (service === "OLLAMA") {renderRecommendedModels(hoststub);} else {const loadModelContainer = document.getElementById("loadModelContainer");@@ -160,7 +160,7 @@loadModelContainer.style.display = "none";}} catch (error) {- handleModelLoadError(provider, error);+ handleModelLoadError(service, error);}}@@ -249,18 +249,18 @@***/function setHoststub() {- // this is called when the user changes the provider- const provider = document.getElementById("provider").value;+ // this is called when the user changes the service+ const service = document.getElementById("service").value;const hoststubInput = document.getElementById("hoststub");const apikeyInput = document.getElementById("apikey");- if (provider === "OLLAMA") {+ if (service === "OLLAMA") {hoststubInput.value = "http://localhost:11434";- } else if (provider === "LMSTUDIO") {+ } else if (service === "LMSTUDIO") {hoststubInput.value = "http://localhost:1234";- } else if (provider === "OPENAI") {+ } else if (service === "OPENAI") {hoststubInput.value = "https://api.openai.com";- } else if (provider === "OPENROUTER") {+ } else if (service === "OPENROUTER") {hoststubInput.value = "https://openrouter.ai/api";} else {hoststubInput.value = "";@@ -268,7 +268,7 @@if (!apikeyInput) return;- if (provider === "OLLAMA" || provider === "LMSTUDIO") {+ if (service === "OLLAMA" || service === "LMSTUDIO") {apikeyInput.disabled = true;apikeyInput.value = "";} else {@@ -312,12 +312,12 @@}}- function renderAvailableModels(provider, payload) {+ function renderAvailableModels(service, payload) {const container = document.getElementById("availableModelsContainer");if (!container) return;- const models = provider === "OLLAMA" ? (payload.models || []) : (payload.data || []);- const getId = provider === "OLLAMA" ? m => m.model : m => m.id;+ const models = service === "OLLAMA" ? (payload.models || []) : (payload.data || []);+ const getId = service === "OLLAMA" ? m => m.model : m => m.id;const rows = [];models.forEach(m => {@@ -330,9 +330,9 @@ranking: info.ranking || "",size: info.size || "",description: info.description || "",- provider: info.provider || "",+ service: info.service || "",license: info.license || "",- renderActions: () => createAvailableModelActionButtons(provider, id)+ renderActions: () => createAvailableModelActionButtons(service, id)});});@@ -351,7 +351,7 @@ranking: info.ranking || "",size: info.size || "",description: info.description || "",- provider: info.provider || "",+ service: info.service || "",license: info.license || "",renderActions: () => createDownloadButton(hoststub, m[0])};@@ -424,8 +424,8 @@container.style.display = "block";}- function createAvailableModelActionButtons(provider, modelId) {- return [createSelectButton(modelId), createDeleteButton(provider, modelId)];+ function createAvailableModelActionButtons(service, modelId) {+ return [createSelectButton(modelId), createDeleteButton(service, modelId)];}function createSelectButton(modelId) {@@ -438,13 +438,13 @@return selectBtn;}- function createDeleteButton(provider, modelId) {+ function createDeleteButton(service, modelId) {const deleteBtn = document.createElement("button");deleteBtn.type = "button";deleteBtn.className = "btn btn-danger btn-sm";deleteBtn.textContent = "Delete";styleActionButton(deleteBtn);- if (provider === "OLLAMA") {+ if (service === "OLLAMA") {deleteBtn.addEventListener("click", () => {const confirmed = window.confirm(`Do you really want to delete model "${modelId}"?`);if (!confirmed) {@@ -478,12 +478,12 @@if (!tbody) return;const hadRowsBeforeInsert = tbody.querySelectorAll("tr").length > 0;- const providerField = document.getElementById("provider");+ const serviceField = document.getElementById("service");const hoststubField = document.getElementById("hoststub");const apikeyField = document.getElementById("apikey");const maxTokenField = document.getElementById("maxtoken");- const provider = providerField ? providerField.value : "";+ const service = serviceField ? serviceField.value : "";const hoststub = hoststubField ? hoststubField.value.trim() : "";const apikey = apikeyField ? apikeyField.value.trim() : "";const maxToken = maxTokenField ? maxTokenField.value : "";@@ -508,7 +508,7 @@}const cells = targetRow.cells;- const values = [provider, modelName, hoststub, apikey, maxToken];+ const values = [service, modelName, hoststub, apikey, maxToken];values.forEach((value, index) => {if (cells[index]) {cells[index].textContent = value || "";@@ -643,10 +643,11 @@return false;}- function collectProductionTablePayload() {+ function persistProductionModels() {+ // read out tableconst tbody = getProductionTableBody();if (!tbody) return [];- const rows = [];+ const production_models_table = [];Array.from(tbody.querySelectorAll("tr")).forEach(row => {if (!row.cells || row.cells.length < PRODUCTION_MODEL_TOTAL_COLUMNS) {return;@@ -662,21 +663,16 @@rowData[columnName] = row.cells[index] ? row.cells[index].textContent.trim() : "";}});- const hasContent = rowData.Provider || rowData.Model || rowData["hoststub"];+ const hasContent = rowData.service || rowData.model || rowData["hoststub"];if (hasContent) {- rows.push(rowData);+ production_models_table.push(rowData);}});- return rows;- }-- function persistProductionModels() {- const table = collectProductionTablePayload();- const payload = { table };++ // push to serverfetch(PRODUCTION_MODEL_SUBMIT_URL, {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify(payload)+ method: "POST", headers: { "Content-Type": "application/json" },+ body: JSON.stringify({ production_models: production_models_table })}).catch(err => {console.error("Failed to persist production models", err);});@@ -724,18 +720,18 @@<h2>LLM Selection</h2><p>- Here you can pick models from a LLM model provider to select them as production model.+ Here you can pick models from a LLM model service to select them as production model.In the "Production Models" - Matrix you can then assign each selected model a function inside YaCy</p><p>- <b>Install your local LLM provider!</b> You need either a local <a href="https://ollama.com/">ollama</a> or <a href="https://lmstudio.ai/">LM Studio</a> instance running on your local host or inside the intranet.+ <b>Install your local LLM service!</b> You need either a local <a href="https://ollama.com/">ollama</a> or <a href="https://lmstudio.ai/">LM Studio</a> instance running on your local host or inside the intranet.</p><form id="llmForm">- <fieldset><legend>Provider</legend>+ <fieldset><legend>Service Selection</legend><dl>- <dt class="TableCellDark">Select a LLM Provider</dt>+ <dt class="TableCellDark">service</dt><dd>- <select name="provider" id="provider" class="form-control" onchange="setHoststub()">+ <select name="service" id="service" class="form-control" onchange="setHoststub()"><option value="OLLAMA" selected="selected">Ollama</option><option value="LMSTUDIO">LMStudio</option><option value="OPENAI">OpenAI</option>@@ -743,15 +739,15 @@</select> This makes a preset to the Hoststub value</dd>- <dt class="TableCellDark">Hoststub</dt>+ <dt class="TableCellDark">hoststub</dt><dd><input type="text" name="hoststub" id="hoststub" value="http://localhost:11434" size="30" maxlength="60" class="form-control"/> you can probably leave this to the default value</dd>- <dt class="TableCellDark">API Key</dt>+ <dt class="TableCellDark">api_key</dt><dd><input type="text" name="apikey" id="apikey" value="" disabled=true size="30" maxlength="60" class="form-control"/> (not required for Ollama or LMStudio)</dd>- <dt class="TableCellDark">Max Token</dt>+ <dt class="TableCellDark">max_tokens</dt><dd><select id="maxtoken" name="maxtoken" class="form-control"><option selected="selected">4096</option>@@ -761,7 +757,7 @@<option>65536</option><option>131072</option><option>262440</option>- </select> You must set the Context Length in the LLM provider to fit to your selected max token; in Ollama you find a Context Length slider in the settings+ </select> You must set the Context Length in the LLM service to fit to your selected max_tokens; in Ollama you find a Context Length slider in the settings</dd><dt> </dt>@@ -781,11 +777,22 @@<fieldset id="productionModelsContainer" style="display: block;"><legend>Production Models</legend><table class="table table-striped" id="productionModelsTable"><thead class="thead-dark">- <tr><td>Provider</td><td>Model</td><td>hoststub</td><td>API Key</td><td>Max Token</td><td>Answers</td><td>Chat</td><td>Translation</td><td>Q/A-Generation</td><td>Classification</td><td>Shortener</td><td>Vision</td><td>Actions</td></tr>+ <tr><td>service</td><td>model</td><td>hoststub</td><td>api_key</td><td>max_tokens</td><td>answers</td><td>chat</td><td>translation</td><td>qa-generation</td><td>classification</td><td>shortener</td><td>vision</td><td>Actions</td></tr></thead><tbody>#{productionmodels}#- <tr class="TableCell#(dark)#Light::Dark#(/dark)#"><td>#[model]#</td><td>#[type]#</td><td>#[size]#</td><td>#[apikey]#</td><td>#[max_token]#</td><td>#[answers]#</td><td>#[chat]#</td><td>#[translation]#</td><td>#[qageneration]#</td><td>#[classification]#</td><td>#[shortener]#</td><td>#[vision]#</td><td></td></tr>+ <tr class="TableCell#(dark)#Light::Dark#(/dark)#">+ <td>#[service]#</td><td>#[model]#</td><td>#[hoststub]#</td><td>#[api_key]#</td><td>#[max_tokens]#</td>+ <td><input type="checkbox" #(answers)#::checked=true#(/answers)#></td>+ <td><input type="checkbox" #(chat)#::checked=true#(/chat)#></td>+ <td><input type="checkbox" #(translation)#::checked=true#(/translation)#></td>+ <td><input type="checkbox" #(qa-generation)#::checked=true#(/qa-generation)#></td>+ <td><input type="checkbox" #(classification)#::checked=true#(/classification)#></td>+ <td><input type="checkbox" #(shortener)#::checked=true#(/shortener)#></td>+ <td><input type="checkbox" #(vision)#::checked=true#(/vision)#></td>+ <td><input type="checkbox" #(chat)#::checked=true#(/chat)#></td>+ <td></td>+ </tr>#{/productionmodels}#</tbody></table>diff --git a/source/net/yacy/htroot/LLMSelection_p.java b/source/net/yacy/htroot/LLMSelection_p.javaindex 2e8a556e9..0a0b43d0a 100644--- a/source/net/yacy/htroot/LLMSelection_p.java+++ b/source/net/yacy/htroot/LLMSelection_p.java@@ -23,18 +23,13 @@package net.yacy.htroot;-import java.io.File;-import java.io.IOException;-import java.util.Date;+import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;+import org.json.JSONTokener;-import net.yacy.cora.date.GenericFormatter;import net.yacy.cora.protocol.RequestHeader;-import net.yacy.data.WorkTables;import net.yacy.search.Switchboard;-import net.yacy.search.SwitchboardConstants;-import net.yacy.search.index.Fulltext;-import net.yacy.search.index.Segment;-import net.yacy.search.schema.CollectionSchema;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;@@ -45,115 +40,69 @@ public class LLMSelection_p {final Switchboard sb = (Switchboard) env;final serverObjects prop = new serverObjects();-- final Segment segment = sb.index;- // we have two counts of document: total number and such that are exportable with status code 200- final long ucount = segment.fulltext().collectionSize();- long ucount200 = ucount;- try {- ucount200 = segment.fulltext().getDefaultConnector().getCountByQuery(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200");- } catch (final IOException e1) {}-- // set default values- prop.put("reload", 0);- prop.put("lurlexport", 0);- prop.putNum("ucount", ucount);- prop.putNum("ucount200", ucount200);-- // show Pack folder contents- int i = 0;- boolean dark = true;- for (final String file: sb.packsInHold()) {- prop.put("packs_" + i + "_file", file);- prop.put("packs_" + i + "_type", "hold");- prop.put("packs_" + i + "_size", new File(sb.packsHoldPath, file).length() / 1024);- prop.put("packs_" + i + "_dark", dark ? "1" : "0");- i++;- dark = !dark;+ String body = post == null ? "" : post.get("BODY", "");+ JSONObject bodyj = new JSONObject();+ if (body.length() > 0) {+ try {+ bodyj = new JSONObject(new JSONTokener(body));+ } catch (JSONException e) {+ // silently catch this+ }}- for (final String file: sb.packsInLoaded()) {- prop.put("packs_" + i + "_file", file);- prop.put("packs_" + i + "_type", "loaded");- prop.put("packs_" + i + "_size", new File(sb.packsLoadedPath, file).length() / 1024);- prop.put("packs_" + i + "_dark", dark ? "1" : "0");- i++;- dark = !dark;+ JSONArray production_models = bodyj.optJSONArray("production_models");+ if (production_models != null) {+ // simply store the model array+ try {+ sb.setConfig("ai.production_models", production_models.toString(0));+ } catch (JSONException e) {+ //e.printStackTrace();+ }}- for (final String file: sb.packsInLive()) {- prop.put("packs_" + i + "_file", file);- prop.put("packs_" + i + "_type", "live");- prop.put("packs_" + i + "_size", new File(sb.packsLivePath, file).length() / 1024);- prop.put("packs_" + i + "_dark", dark ? "1" : "0");- i++;- dark = !dark;+ /*+ {"production_models":[{+ "service":"OLLAMA",+ "model":"hf.co\/janhq\/Jan-v1-edge-gguf:Q4_K_M",+ "hoststub":"http:\/\/localhost:11434",+ "api_key":"",+ "max_tokens":"4096",+ "answers":true,+ "chat":true,+ "translation":true,+ "qa-generation":true,+ "classification":true,+ "tldr-shortener":true,+ "vision":true+ }]}+ */++ // generate table for production_models+ String pms = sb.getConfig("ai.production_models", "[]");+ try {+ production_models = new JSONArray(new JSONTokener(pms));+ for (int i = 0; i < production_models.length(); i++) {+ JSONObject row = production_models.getJSONObject(i);+ prop.put("productionmodels_" + i + "_service", row.optString("service", "OLLAMA"));+ prop.put("productionmodels_" + i + "_model", row.optString("model", ""));+ prop.put("productionmodels_" + i + "_hoststub", row.optString("hoststub", ""));+ prop.put("productionmodels_" + i + "_api_key", row.optString("api_key", ""));+ prop.put("productionmodels_" + i + "_max_tokens", row.optString("max_tokens", "4096"));+ prop.put("productionmodels_" + i + "_answers", row.optBoolean("answers", false));+ prop.put("productionmodels_" + i + "_chat", row.optBoolean("chat", false));+ prop.put("productionmodels_" + i + "_translation", row.optBoolean("translation", false));+ prop.put("productionmodels_" + i + "_qa-generation", row.optBoolean("qa-generation", false));+ prop.put("productionmodels_" + i + "_classification", row.optBoolean("classification", false));+ prop.put("productionmodels_" + i + "_tldr-shortener", row.optBoolean("tldr-shortener", false));+ prop.put("productionmodels_" + i + "_vision", row.optBoolean("vision", false));+ }+ prop.put("productionmodels", production_models.length());+ } catch (JSONException e) {+ e.printStackTrace();}- prop.put("packs", i);if (post == null || env == null) {return prop; // nothing to do}- if (post.containsKey("lurlexport")) {- try {- // parse format- Fulltext.ExportFormat format = Fulltext.ExportFormat.elasticsearch;- final String fname = post.get("format", "full-elasticsearch");- final boolean dom = fname.startsWith("dom"); // if dom== false complete urls are exported, otherwise only the domain- final boolean text = fname.startsWith("text");- if (fname.endsWith("rss")) format = Fulltext.ExportFormat.rss;- if (fname.endsWith("solr")) format = Fulltext.ExportFormat.solr;- if (fname.endsWith("elasticsearch")) format = Fulltext.ExportFormat.elasticsearch;-- final String filter = post.get("exportfilter", ".*");- String query = post.get("exportquery", "*:*");- final String collection = post.get("collection", "user");- query += " AND " + CollectionSchema.collection_sxt.getSolrFieldName() + ":\"" + collection + "\"";-- // store this call as api call: we do this even if there is a chance that it fails because recurring calls may do not fail- sb.tables.recordAPICall(post, "IndexPackGenerator_p.html", WorkTables.TABLE_API_TYPE_DUMP, "PackGenerator, q=" + query);-- // start the export- /*- Tier Tags:- | Tier | Size | Notes |- |-----------|-----------|--------------------|- | common | ≤ 1 GB | IndexPackGenerator |- | uncommon | 1–5 GB | large web crawls |- | rare | 5–50 GB | custom parser |- | epic | 50–200 GB | special infra |- | legendary | any | human curation |- */- final long now = System.currentTimeMillis();- final long doccount = sb.index.fulltext().getDefaultConnector().getCountByQuery(query);- if (doccount == 0) throw new IOException("number of exported documents == 0");- final String category = post.get("category", "scroll"); // core, scroll, codex, gem, fiction, map, echo, spirit, vault- final String tier = "common"; // common, uncommon, rare, epic, legendary, legendary- final String origin = "web"; // web, synth,- String slug = post.get("slug", "export").trim().replaceAll(" ", "-");- if (slug.isEmpty()) slug = "export";-- // if collection is not user, the slug is the collection name- if (!"user".equals(collection)) {- slug = collection.trim().replaceAll(" ", "-");- }- // we can not construct the file name- final String filename =- SwitchboardConstants.YACY_PACK_PREFIX +- category + "-" + tier + "-" + origin + "_" +- slug + "_" +- GenericFormatter.SHORT_DAY_FORMATTER.format(new Date(now));- // file name schema: YaCyPack_<category>-<tier>-<origin>_<slug>_<YYMMDD>.jsonlist- // possible storage paths are: hold, load, loaded, unload, live; we use hold here, loaded would also be correct- } catch (final IOException e) {- prop.put("lurlexporterror", 1);- prop.put("lurlexporterror_exportfile", "-no export-");- prop.put("lurlexporterror_exportfailmsg", e.getMessage());- return prop;- }- }-- // insert constants- prop.putNum("ucount", ucount);// return rewrite propertiesreturn prop;}