Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
added tool calling abilities to yacychat.html and added a tool handler to RAGProxyServlet; also provided a large set of basic tools to handle calculations, date understanding, unit conversion, web fetch, number parsing, self reflection and more
Michael Peter Christen16a91ab12ccb
diff --git a/htroot/yacychat.html b/htroot/yacychat.htmlindex 50d77f6e3..76ae0ebf5 100644--- a/htroot/yacychat.html+++ b/htroot/yacychat.html@@ -83,6 +83,21 @@color: #ffffff;border: 1px solid #2c3e50;}++ /* --- TOOL: Tool Call Output --- */+ .chat-turn.tool {+ background-color: #f0f7e8;+ border-left-width: 5px;+ border-left-style: solid;+ border-color: #b7d7a8;+ color: #24421a;+ }++ .chat-turn.tool legend {+ background: #3d6b2f;+ color: #ffffff;+ border: 1px solid #3d6b2f;+ }/* --- SYSTEM: Log Entry Style --- */.chat-turn.system {@@ -147,6 +162,68 @@font-size: 0.85rem;}+ .toolcalls-popover {+ position: absolute;+ z-index: 20;+ top: 24px;+ right: 8px;+ min-width: 280px;+ max-width: min(560px, calc(100vw - 40px));+ max-height: 380px;+ overflow: auto;+ border: 1px solid #c3cbd9;+ border-radius: 6px;+ background: #eef2f7;+ box-shadow: 3px 3px 0px rgba(0,0,0,0.15);+ padding: 8px 10px;+ }++ .toolcalls-popover-header {+ font-weight: 700;+ margin-bottom: 6px;+ }++ .toolcalls-list {+ margin-top: 6px;+ display: flex;+ flex-direction: column;+ gap: 6px;+ }++ .toolcall-item {+ border: 1px solid #d6dbe5;+ border-radius: 4px;+ background: #f5f7fb;+ padding: 6px 8px;+ }++ .toolcall-name {+ font-weight: 700;+ margin-bottom: 4px;+ }++ .toolcall-args {+ margin: 0;+ white-space: pre-wrap;+ word-break: break-word;+ font-size: 0.95rem;+ line-height: 1.4;+ }++ .toolcall-result-title {+ margin-top: 6px;+ margin-bottom: 4px;+ font-weight: 700;+ }++ .toolcall-result {+ margin: 0;+ white-space: pre-wrap;+ word-break: break-word;+ font-size: 0.95rem;+ line-height: 1.4;+ }+.chat-body.markdown h1,.chat-body.markdown h2,.chat-body.markdown h3,@@ -316,7 +393,8 @@.attachment-button,.search-button,.copy-button,- .user-trim-button {+ .user-trim-button,+ .toolcalls-button {width: 24px;height: 24px;border: none;@@ -335,7 +413,8 @@.attachment-button:hover,.search-button:hover,.copy-button:hover,- .user-trim-button:hover {+ .user-trim-button:hover,+ .toolcalls-button:hover {background: #c3cbd9;}@@ -374,6 +453,20 @@right: 8px;}+ .toolcalls-button {+ position: static;+ }++ .toolcall-toolbar {+ position: absolute;+ top: -6px;+ right: 36px;+ display: inline-flex;+ flex-direction: row;+ gap: 4px;+ align-items: center;+ }+.user-delete-button {position: absolute;top: -6px;@@ -775,6 +868,7 @@const ALLOWED_EXTENSIONS = new Set([...IMAGE_EXTENSIONS, ...TEXT_EXTENSIONS]);const FILE_ACCEPT_TYPES = [...ALLOWED_MIME_TYPES, ...ALLOWED_EXTENSIONS].join(',');+if (dom.fileInput) {dom.fileInput.setAttribute('accept', FILE_ACCEPT_TYPES);}@@ -966,18 +1060,31 @@return result;}+ function stripToolProtocolMarkup(text) {+ if (!text || typeof text !== 'string') return text || '';+ // Remove legacy/model-emitted tool protocol snippets like:+ // <function=name>+ // <parameter=key>+ // value+ let cleaned = text.replace(/^\s*<function=[^>\n]+>\s*$[\s\S]*?(?=^\s*$|$)/gim, '');+ cleaned = cleaned.replace(/^\s*<parameter=[^>\n]+>\s*$[\s\S]*?(?=^\s*$|$)/gim, '');+ // Collapse excessive blank lines introduced by stripping.+ cleaned = cleaned.replace(/\n{3,}/g, '\n\n');+ return cleaned;+ }+function stripThinkFromContent(content) {- if (typeof content === 'string') return stripThinkBlocks(content);+ if (typeof content === 'string') return stripToolProtocolMarkup(stripThinkBlocks(content));if (Array.isArray(content)) {return content.map(part => {if (part?.type === 'text' && typeof part.text === 'string') {- return { ...part, text: stripThinkBlocks(part.text) };+ return { ...part, text: stripToolProtocolMarkup(stripThinkBlocks(part.text)) };}return part;});}if (content && typeof content === 'object' && typeof content.text === 'string') {- return { ...content, text: stripThinkBlocks(content.text) };+ return { ...content, text: stripToolProtocolMarkup(stripThinkBlocks(content.text)) };}return content;}@@ -1143,7 +1250,7 @@}function appendMessage(role, text, opts = {}) {- const { skipScroll = false, skipVisibilityUpdate = false, attachment = null, messageIndex = undefined } = opts;+ const { skipScroll = false, skipVisibilityUpdate = false, attachment = null, messageIndex = undefined, toolCalls = null, toolResults = null } = opts;const entry = document.createElement('fieldset');entry.className = `chat-turn ${role}`;if (role === 'assistant' || role === 'user') {@@ -1154,6 +1261,8 @@legend.textContent = 'Assistant';} else if (role === 'user') {legend.textContent = 'User';+ } else if (role === 'tool') {+ legend.textContent = 'Tool';} else {legend.textContent = 'System';}@@ -1172,7 +1281,6 @@attachmentContainer.appendChild(createAttachmentChip(attachment));entry.appendChild(attachmentContainer);}-dom.messages.appendChild(entry);if (typeof opts.messageIndex === 'number') {entry.dataset.messageIndex = String(opts.messageIndex);@@ -1185,6 +1293,56 @@copyBtn.innerHTML = '<span class="glyphicon glyphicon-copy" aria-hidden="true"></span>';copyBtn.addEventListener('click', () => copyAssistant(body));entry.appendChild(copyBtn);+ if (Array.isArray(toolCalls) && toolCalls.length > 0) {+ const toolbar = document.createElement('div');+ toolbar.className = 'toolcall-toolbar';+ const panels = [];+ const resultsById = new Map();+ const resultsByName = new Map();+ if (Array.isArray(toolResults)) {+ for (const result of toolResults) {+ if (!result) continue;+ if (result.id) resultsById.set(result.id, result);+ if (result.tool_call_id) resultsById.set(result.tool_call_id, result);+ const rname = result?.name || result?.function?.name;+ if (rname && !resultsByName.has(rname)) resultsByName.set(rname, result);+ }+ }++ for (let i = 0; i < toolCalls.length; i++) {+ const call = toolCalls[i];+ if (!call) continue;+ const fnName = call?.function?.name || call?.name || 'tool';+ const matchedResult = (call.id && resultsById.get(call.id)) || resultsByName.get(fnName) || null;++ const toolBtn = document.createElement('button');+ toolBtn.type = 'button';+ toolBtn.className = 'toolcalls-button';+ toolBtn.title = `Tool: ${fnName}`;+ toolBtn.innerHTML = '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span>';++ const toolPanel = createToolCallPanel(call, matchedResult, toolCalls.length, i);+ toolPanel.style.display = 'none';+ toolPanel.addEventListener('click', evt => evt.stopPropagation());+ entry.appendChild(toolPanel);+ panels.push(toolPanel);++ toolBtn.addEventListener('click', event => {+ event.stopPropagation();+ const open = toolPanel.style.display !== 'none';+ for (const panel of panels) panel.style.display = 'none';+ toolPanel.style.display = open ? 'none' : 'block';+ });+ toolbar.appendChild(toolBtn);+ }+ entry.appendChild(toolbar);++ document.addEventListener('click', evt => {+ if (!entry.contains(evt.target)) {+ for (const panel of panels) panel.style.display = 'none';+ }+ });+ }}if (role === 'user' && typeof messageIndex === 'number') {const trimBtn = document.createElement('button');@@ -1496,6 +1654,9 @@const reader = response.body.getReader();const decoder = new TextDecoder('utf-8');let assistantText = '';+ let roundToolCalls = [];+ let collectedToolCalls = [];+ let collectedToolResults = [];let sawFirstDelta = false;let focusShown = false;let searchApplied = false;@@ -1557,7 +1718,31 @@try {const parsed = JSON.parse(clean);applySearchAttachment(parsed);- const delta = parsed?.choices?.[0]?.delta?.content;+ if (Array.isArray(parsed['tool-results']) && parsed['tool-results'].length > 0) {+ collectedToolResults = collectedToolResults.concat(normalizeToolResults(parsed['tool-results']));+ }+ const choice = parsed?.choices?.[0];+ const delta = choice?.delta;+ const finishReason = choice?.finish_reason;+ const toolDelta = delta?.tool_calls;+ if (toolDelta) {+ roundToolCalls = mergeToolCalls(roundToolCalls, toolDelta);+ }+ if (delta?.function_call) {+ roundToolCalls = mergeToolCalls(roundToolCalls, [{+ index: 0,+ type: 'function',+ function: delta.function_call+ }]);+ }+ if (finishReason === 'tool_calls' || finishReason === 'function_call') {+ const finalized = normalizeToolCalls(roundToolCalls);+ if (finalized.length) {+ collectedToolCalls = collectedToolCalls.concat(finalized);+ }+ roundToolCalls = [];+ }+ const contentDelta = delta?.content;if (delta) {if (!sawFirstDelta) {sawFirstDelta = true;@@ -1567,7 +1752,10 @@onFirstToken();}}- assistantText += delta;+ if (contentDelta) {+ assistantText += contentDelta;+ assistantText = stripToolProtocolMarkup(assistantText);+ }if (!thinkAutoClosed && hasThinkCloseOutsideFences(assistantText)) {thinkAutoClosed = true;if (assistantNode) {@@ -1637,7 +1825,22 @@: explicitThinkOpen === 'false'? false: (thinkAutoClosed && !thinkAutoCloseFired);- state.messages.push({ role: 'assistant', content: assistantText, thinkOpen: storedThinkOpen });+ const normalizedToolCalls = normalizeToolCalls(collectedToolCalls.concat(roundToolCalls));+ const assistantMessage = {+ role: 'assistant',+ content: assistantText,+ thinkOpen: storedThinkOpen+ };+ if (normalizedToolCalls.length) {+ assistantMessage.tool_calls = normalizedToolCalls;+ if (collectedToolResults.length) {+ assistantMessage.tool_results = collectedToolResults;+ }+ assistantMessage.display = assistantText && assistantText.trim()+ ? assistantText+ : `Tool call: ${normalizedToolCalls.map(call => call?.function?.name || 'tool').join(', ')}`;+ }+ state.messages.push(assistantMessage);console.debug('[yacychat] response chars', assistantText.length, 'total ms', Math.round(performance.now() - requestStart));state.assistantSeen = true;persistConversation();@@ -1691,12 +1894,21 @@}if (message.role === 'assistant') {if (textOnly || typeof content === 'string') {- content = stripThinkBlocks(String(content || ''));+ content = stripToolProtocolMarkup(stripThinkBlocks(String(content || '')));} else {content = stripThinkFromContent(content);}}const sanitized = { role: message.role, content };+ if (message.tool_calls) {+ sanitized.tool_calls = message.tool_calls;+ }+ if (message.tool_call_id) {+ sanitized.tool_call_id = message.tool_call_id;+ }+ if (message.name) {+ sanitized.name = message.name;+ }if (message.search === true) {sanitized.search = SEARCH_DEFAULTS.local;} else if (message.search === SEARCH_DEFAULTS.local || message.search === SEARCH_DEFAULTS.global) {@@ -1726,6 +1938,12 @@return '';}+ function messageDisplayText(message) {+ if (!message) return '';+ if (typeof message.display === 'string') return message.display;+ return messageText(message.content);+ }+function renderConversation(options = {}) {const { skipScroll = false } = options;closeAttachmentPopover();@@ -1734,7 +1952,7 @@for (let i = 0; i < state.messages.length; i++) {const msg = state.messages[i];if (msg.role === 'system' && !state.showSystem) continue;- appendMessage(msg.role, messageText(msg.content), { skipScroll: true, skipVisibilityUpdate: true, messageIndex: i, attachment: msg.attachment });+ appendMessage(msg.role, messageDisplayText(msg), { skipScroll: true, skipVisibilityUpdate: true, messageIndex: i, attachment: msg.attachment, toolCalls: msg.tool_calls, toolResults: msg.tool_results });}updateClearChatVisibility();updateSystemToggleButton();@@ -1743,6 +1961,119 @@}}+ function normalizeToolCalls(calls) {+ if (!Array.isArray(calls)) return [];+ const stamp = Date.now();+ return calls+ .filter(Boolean)+ .map((call, index) => {+ const fn = call.function || {};+ return {+ id: call.id || `toolcall_${stamp}_${index}`,+ type: call.type || 'function',+ function: {+ name: fn.name || call.name || 'unknown',+ arguments: typeof fn.arguments === 'string' ? fn.arguments : (call.arguments || '')+ }+ };+ });+ }++ function normalizeToolResults(results) {+ if (!Array.isArray(results)) return [];+ const stamp = Date.now();+ return results+ .filter(Boolean)+ .map((result, index) => ({+ id: result.id || result.tool_call_id || `toolres_${stamp}_${index}`,+ tool_call_id: result.tool_call_id || result.id || '',+ name: result.name || result?.function?.name || 'unknown',+ content: typeof result.content === 'string'+ ? result.content+ : (result?.function?.content ? String(result.function.content) : '')+ }));+ }++ function mergeToolCalls(existing, deltaCalls) {+ const result = Array.isArray(existing) ? [...existing] : [];+ if (!Array.isArray(deltaCalls)) return result;+ for (const delta of deltaCalls) {+ if (!delta) continue;+ const index = readToolCallIndex(delta);+ if (index === null || index < 0) continue;+ if (!result[index]) {+ result[index] = { id: '', type: delta.type || 'function', function: { name: '', arguments: '' } };+ }+ const target = result[index];+ if (delta.id) target.id = delta.id;+ if (delta.type) target.type = delta.type;+ if (delta.function?.name) target.function.name = delta.function.name;+ if (typeof delta.function?.arguments === 'string' && delta.function.arguments.length > 0) {+ target.function.arguments = (target.function.arguments || '') + delta.function.arguments;+ }+ }+ return result;+ }++ function readToolCallIndex(delta) {+ if (!delta || typeof delta !== 'object' || !Object.prototype.hasOwnProperty.call(delta, 'index')) {+ return null;+ }+ return Number.isInteger(delta.index) ? delta.index : null;+ }++ function formatToolArguments(raw) {+ if (typeof raw !== 'string' || !raw.trim()) return '(no arguments)';+ try {+ return JSON.stringify(JSON.parse(raw), null, 2);+ } catch (err) {+ return raw;+ }+ }++ function createToolCallPanel(call, result, totalCount, index) {+ const panel = document.createElement('div');+ panel.className = 'toolcalls-popover';+ panel.style.right = `${36 + (index * 28)}px`;++ const header = document.createElement('div');+ header.className = 'toolcalls-popover-header';+ header.textContent = `Tool ${index + 1}/${totalCount}`;+ panel.appendChild(header);++ const list = document.createElement('div');+ list.className = 'toolcalls-list';+ const item = document.createElement('div');+ item.className = 'toolcall-item';++ const fnName = call?.function?.name || call?.name || 'unknown';+ const name = document.createElement('div');+ name.className = 'toolcall-name';+ name.textContent = fnName;+ item.appendChild(name);++ const args = document.createElement('pre');+ args.className = 'toolcall-args';+ args.textContent = formatToolArguments(call?.function?.arguments || call?.arguments || '');+ item.appendChild(args);++ if (result) {+ const responseTitle = document.createElement('div');+ responseTitle.className = 'toolcall-result-title';+ responseTitle.textContent = 'Response';+ item.appendChild(responseTitle);++ const responseBody = document.createElement('pre');+ responseBody.className = 'toolcall-result';+ responseBody.textContent = formatToolArguments(result?.content || result?.function?.content || '');+ item.appendChild(responseBody);+ }++ list.appendChild(item);+ panel.appendChild(list);+ return panel;+ }+function persistConversation() {try {const messages = state.messages.map(msg => {diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.javaindex 7987cbb29..cf12265ef 100644--- a/source/net/yacy/ai/LLM.java+++ b/source/net/yacy/ai/LLM.java@@ -1,5 +1,5 @@/**- * LLMEndpoint+ * LLM* Copyright 2024 by Michael Peter Christen* First released 17.05.2024 at https://yacy.net*diff --git a/source/net/yacy/ai/LLMSwarm.java b/source/net/yacy/ai/LLMSwarm.javaindex 9aa3286b2..67508c236 100644--- a/source/net/yacy/ai/LLMSwarm.java+++ b/source/net/yacy/ai/LLMSwarm.java@@ -1,5 +1,5 @@/**- * OllamaSwarm.java+ * LLMSwarm.java* Copyright 2025 by Michael Peter Christen* First released 01.06.2025 at https://yacy.net*diff --git a/source/net/yacy/ai/RAGAugmentor.java b/source/net/yacy/ai/RAGAugmentor.javanew file mode 100644index 000000000..d1dbb18db--- /dev/null+++ b/source/net/yacy/ai/RAGAugmentor.java@@ -0,0 +1,614 @@+/**+ * RAGAugmentor+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai;++import java.io.IOException;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Collection;+import java.util.Comparator;+import java.util.HashMap;+import java.util.HashSet;+import java.util.Iterator;+import java.util.LinkedHashSet;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.stream.Collectors;++import org.apache.solr.client.solrj.SolrQuery;+import org.apache.solr.common.SolrDocument;+import org.apache.solr.common.SolrDocumentList;+import org.apache.solr.common.SolrException;+import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.cora.document.analysis.Classification;+import net.yacy.cora.document.id.DigestURL;+import net.yacy.cora.document.id.MultiProtocolURL;+import net.yacy.cora.federate.solr.SolrType;+import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;+import net.yacy.cora.federate.yacy.CacheStrategy;+import net.yacy.cora.lod.vocabulary.Tagging;+import net.yacy.cora.protocol.ClientIdentification;+import net.yacy.cora.util.ConcurrentLog;+import net.yacy.kelondro.data.meta.URIMetadataNode;+import net.yacy.search.Switchboard;+import net.yacy.search.SwitchboardConstants;+import net.yacy.search.query.QueryGoal;+import net.yacy.search.query.QueryModifier;+import net.yacy.search.query.QueryParams;+import net.yacy.search.query.SearchEvent;+import net.yacy.search.query.SearchEventCache;+import net.yacy.search.ranking.RankingProfile;+import net.yacy.search.schema.CollectionSchema;+import net.yacy.search.snippet.TextSnippet;++/**+ * Utility methods that enrich prompts/responses with search-derived context for+ * Retrieval-Augmented Generation (RAG).+ * <p>+ * This class provides:+ * <ul>+ * <li>local Solr-backed search result extraction</li>+ * <li>global YaCy network search extraction</li>+ * <li>markdown condensation of search results</li>+ * <li>token intersection helpers for query boosting</li>+ * <li>snippet scoring/selection utilities</li>+ * </ul>+ */+public final class RAGAugmentor {++ /**+ * Utility class; not instantiable.+ */+ private RAGAugmentor() {}++ /**+ * Executes local index search with default boost terms.+ *+ * @param query query string+ * @param count max number of results+ * @param includeSnippet include text snippet field in response objects+ * @return JSON array with {@code url,title[,text]} entries+ */+ public static JSONArray searchResults(String query, int count, final boolean includeSnippet) {+ return searchResults(query, count, includeSnippet, new LinkedHashSet<>());+ }++ /**+ * Executes local Solr search with optional dynamic boost terms.+ *+ * @param query query string+ * @param count max number of results+ * @param includeSnippet include text snippets from indexed text field+ * @param boostTerms optional overlap terms used to bias ranking+ * @return JSON array with normalized search result objects+ */+ public static JSONArray searchResults(String query, int count, final boolean includeSnippet, final Set<String> boostTerms) {+ final JSONArray results = new JSONArray();+ if (query == null || query.length() == 0 || count == 0) return results;+ Switchboard sb = Switchboard.getSwitchboard();+ EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector();+ final SolrQuery params = new SolrQuery();+ // Base query and parser setup.+ params.setQuery(query);+ params.set("defType", "edismax");+ // Static field boosts favor title/headings over body text.+ params.set("qf",+ CollectionSchema.title.getSolrFieldName() + "^3 " ++ CollectionSchema.text_t.getSolrFieldName() + "^1 " ++ CollectionSchema.sku.getSolrFieldName() + "^0.5 " ++ CollectionSchema.h1_txt.getSolrFieldName() + "^2");+ params.set("pf",+ CollectionSchema.title.getSolrFieldName() + "^5 " ++ CollectionSchema.text_t.getSolrFieldName() + "^2");+ final List<String> bqParts = new ArrayList<>();+ if (boostTerms != null && !boostTerms.isEmpty()) {+ // Apply weak boosts to overlap terms, keeping lexical query dominant.+ for (String term : boostTerms) {+ if (term == null || term.isEmpty()) continue;+ bqParts.add(CollectionSchema.title.getSolrFieldName() + ":" + term + "^0.5");+ bqParts.add(CollectionSchema.h1_txt.getSolrFieldName() + ":" + term + "^0.4");+ bqParts.add(CollectionSchema.text_t.getSolrFieldName() + ":" + term + "^0.2");+ }+ }+ // Slightly prefer archive/container file extensions in this use case.+ bqParts.add("(" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(zip rar 7z tar gz bz2 xz tgz))^0.1");+ params.set("bq", String.join(" ", bqParts));+ params.setRows(count);+ params.setStart(0);+ params.setFacet(false);+ params.clearSorts();+ // Fetch only fields needed for tool / markdown rendering.+ params.setFields(+ CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName(),+ CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.keywords.getSolrFieldName(), CollectionSchema.synonyms_sxt.getSolrFieldName(),+ CollectionSchema.h1_txt.getSolrFieldName(), CollectionSchema.h2_txt.getSolrFieldName(), CollectionSchema.h3_txt.getSolrFieldName(),+ CollectionSchema.h4_txt.getSolrFieldName(), CollectionSchema.h5_txt.getSolrFieldName(), CollectionSchema.h6_txt.getSolrFieldName()+ );+ params.setIncludeScore(true);+ params.set("df", CollectionSchema.text_t.getSolrFieldName());++ try {+ final SolrDocumentList sdl = connector.getDocumentListByParams(params);+ Iterator<SolrDocument> i = sdl.iterator();+ while (i.hasNext()) {+ try {+ SolrDocument doc = i.next();+ final JSONObject result = new JSONObject(true);+ String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());+ result.put("url", url == null ? "" : url.trim());+ String title = getOneString(doc, CollectionSchema.title);+ result.put("title", title == null ? "" : title.trim());+ if (includeSnippet) {+ // Use indexed text body as quick snippet source.+ String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());+ result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));+ }+ results.put(result);+ } catch (JSONException e) {+ }+ }+ return results;+ } catch (SolrException | IOException e) {+ return results;+ }+ }++ /**+ * Renders search results as compact markdown context using local-search mode.+ *+ * @param query query string+ * @param count max number of search rows+ * @param global when true, use global YaCy search+ * @return markdown context block+ */+ public static String searchResultsAsMarkdown(String query, int count, boolean global) {+ return searchResultsAsMarkdown(query, count, global, new LinkedHashSet<>());+ }++ /**+ * Renders search results as compact markdown and applies snippet ranking to+ * reduce noise.+ *+ * @param query query string+ * @param count max number of search rows+ * @param global when true, use global YaCy search; otherwise local Solr+ * @param boostTerms optional local-search boost terms+ * @return markdown formatted context used in downstream prompt augmentation+ */+ public static String searchResultsAsMarkdown(String query, int count, boolean global, final Set<String> boostTerms) {+ final long searchStart = System.currentTimeMillis();+ JSONArray searchResults = global ? searchResultsGlobal(query, count, true) : searchResults(query, count, true, boostTerms);+ ConcurrentLog.info("RAGProxy", "searchResults=" + searchResults.length() + " global=" + global + " searchMs=" + (System.currentTimeMillis() - searchStart));+ StringBuilder sb = new StringBuilder();++ // Convert raw rows into scoreable snippet candidates.+ List<Snippet> results = new ArrayList<>();+ for (int i = 0; i < searchResults.length(); i++) {+ try {+ JSONObject r = searchResults.getJSONObject(i);+ String title = r.optString("title", "");+ String url = r.optString("url", "");+ String text = r.optString("text", "");+ if (title.isEmpty()) title = url;+ if (text.isEmpty()) text = title;+ if (title.length() > 0 && text.length() > 0) {+ Snippet snippet = new Snippet(query, text, url, title, 256);+ if (snippet.getText().length() > 0) results.add(snippet);+ }+ } catch (JSONException e) {}+ }++ // Lower score is better with the current tf-idf based chunk scorer.+ results.sort(Comparator.comparingDouble(Snippet::getScore));+ // Keep top half to avoid overloading the model context window.+ int limit = results.size() / 2;+ if (results.size() > 0 && limit == 0) limit = 1;+ for (int i = 0; i < limit; i++) {+ Snippet snippet = results.get(i);+ sb.append("## ").append(snippet.getTitle()).append("\n");+ sb.append(snippet.text).append("\n");+ if (snippet.getURL().length() > 0) sb.append("Source: ").append(snippet.getURL()).append("\n");+ sb.append("\n\n");+ }++ ConcurrentLog.info("RAGProxy", "markdownChars=" + sb.length() + " snippetCount=" + results.size());+ return sb.toString();+ }++ /**+ * Executes a global/distributed YaCy search event and maps results into a+ * compact JSON format.+ *+ * @param query query string+ * @param count max number of results+ * @param includeSnippet include snippet text when available+ * @return JSON array with normalized result objects+ */+ public static JSONArray searchResultsGlobal(String query, int count, final boolean includeSnippet) {+ final JSONArray results = new JSONArray();+ if (query == null || query.length() == 0 || count == 0) return results;+ final Switchboard sb = Switchboard.getSwitchboard();+ final RankingProfile ranking = sb.getRanking();+ final int timezoneOffset = 0;+ final QueryModifier modifier = new QueryModifier(timezoneOffset);+ // Parse modifiers and normalize effective query string.+ String querystring = modifier.parse(query);+ if (querystring.length() == 0) querystring = query == null ? "" : query.trim();+ if (querystring.length() == 0) return results;+ final QueryGoal qg = new QueryGoal(querystring);+ // Construct a standard text-domain global query.+ final QueryParams theQuery = new QueryParams(+ qg,+ modifier,+ 0,+ "",+ Classification.ContentDomain.TEXT,+ "",+ timezoneOffset,+ new HashSet<Tagging.Metatag>(),+ CacheStrategy.IFFRESH,+ count,+ 0,+ ".*",+ null,+ null,+ QueryParams.Searchdom.GLOBAL,+ null,+ true,+ DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")),+ MultiProtocolURL.TLD_any_zone_filter,+ null,+ false,+ sb.index,+ ranking,+ ClientIdentification.yacyIntranetCrawlerAgent.userAgent(),+ 0.0d,+ 0.0d,+ 0.0d,+ sb.getConfigSet("search.navigation"));+ final SearchEvent theSearch = SearchEventCache.getEvent(+ theQuery,+ sb.peers,+ sb.tables,+ (sb.isRobinsonMode()) ? sb.clusterhashes : null,+ false,+ sb.loader,+ (int) sb.getConfigLong(+ SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER,+ sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_DEFAULT, 10)),+ sb.getConfigLong(+ SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,+ sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)));+ final long timeout = sb.getConfigLong(+ SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,+ sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000));+ // Wait until remote feeds are done (or timeout), then stabilize ordering.+ waitForFeedingAndResort(theSearch, timeout);+ for (int i = 0; i < count; i++) {+ URIMetadataNode node = theSearch.oneResult(i, timeout);+ if (node == null) break;+ try {+ final JSONObject result = new JSONObject(true);+ result.put("url", node.urlstring());+ result.put("title", node.title());+ if (includeSnippet) {+ // Prefer direct snippet; fall back to textSnippet, description, then text field.+ String text = node.snippet();+ if (text == null || text.isEmpty()) {+ TextSnippet snippet = node.textSnippet();+ if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) text = snippet.getLineRaw();+ }+ if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.description_txt.getSolrFieldName()));+ if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.text_t.getSolrFieldName()));+ result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));+ }+ results.put(result);+ } catch (JSONException e) {+ }+ }+ return results;+ }++ /**+ * Uses an LLM list schema prompt to compute likely discriminative search+ * terms for a user prompt.+ *+ * @param llm configured LLM backend+ * @param model target model name+ * @param prompt user prompt to analyze+ * @return space-separated lowercase term list or {@code null} on failure+ */+ public static String searchWordsForPrompt(LLM llm, String model, String prompt) {+ final String question = prompt == null ? "" : prompt;+ if (llm == null || model == null || model.isEmpty()) return null;+ try {+ LLM.Context context = new LLM.Context("\n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.");+ context.addPrompt(question);+ Set<String> singlewords = new LinkedHashSet<>();+ String[] a = LLM.stringsFromChat(llm.chat(model, context, LLM.listSchema, 200));+ if (a == null || a.length == 0) return null;+ for (String s: a) {+ if (s == null) continue;+ // Flatten model output into unique lowercased tokens.+ for (String t: s.split(" ")) if (!t.isEmpty()) singlewords.add(t.toLowerCase());+ }+ if (singlewords.isEmpty()) return null;+ StringBuilder query = new StringBuilder();+ for (String s: singlewords) query.append(s).append(' ');+ String querys = query.toString().trim();+ if (querys.length() == 0) return null;+ return querys;+ } catch (IOException | JSONException e) {+ e.printStackTrace();+ return null;+ }+ }++ /**+ * Computes token overlap between original prompt and computed query terms.+ *+ * @param originalPrompt raw user prompt+ * @param computedQuery generated query terms+ * @param maxTerms max terms to keep; {@code <=0} means unlimited+ * @return cleaned ordered overlap set+ */+ public static Set<String> intersectTokens(String originalPrompt, String computedQuery, int maxTerms) {+ Set<String> promptTerms = querySet(originalPrompt == null ? "" : originalPrompt);+ Set<String> queryTerms = querySet(computedQuery == null ? "" : computedQuery);+ Set<String> intersection = new LinkedHashSet<>();+ for (String term : promptTerms) {+ if (!queryTerms.contains(term)) continue;+ final String cleaned = cleanToken(term);+ if (cleaned.isEmpty()) continue;+ intersection.add(cleaned);+ if (maxTerms > 0 && intersection.size() >= maxTerms) break;+ }+ return intersection;+ }++ /**+ * Splits text into sentence-aware chunks around a target max length.+ *+ * @param text source text+ * @param len approximate chunk length+ * @return ordered chunks+ */+ public static List<String> slicer(String text, int len) {+ List<String> result = new ArrayList<>();+ if (text == null || len <= 0) return result;++ int start = 0;+ while (start < text.length()) {+ int end = Math.min(start + len, text.length());+ // Extend to sentence boundary when possible.+ while (end < text.length()) {+ char ch = text.charAt(end - 1);+ if ((ch == '.' || ch == '?' || ch == '!') && Character.isWhitespace(text.charAt(end))) break;+ end++;+ }+ result.add(text.substring(start, end));+ start = end;+ }+ return result;+ }++ /**+ * Converts a query string into a normalized token set.+ *+ * @param query raw query text+ * @return lowercase token set+ */+ private static Set<String> querySet(String query) {+ return Arrays.stream(query.trim().toLowerCase().split("\\s+"))+ .map(String::toLowerCase)+ .filter(word -> !word.isEmpty())+ .collect(Collectors.toSet());+ }++ /**+ * Removes non-alphanumeric characters and enforces minimal token length.+ *+ * @param term source token+ * @return cleaned lowercase token or empty string+ */+ private static String cleanToken(String term) {+ if (term == null) return "";+ String cleaned = term.replaceAll("[^A-Za-z0-9]", "");+ if (cleaned.length() < 2) return "";+ return cleaned.toLowerCase();+ }++ /**+ * Truncates snippet text to a maximum character count.+ *+ * @param text input text+ * @param maxChars limit+ * @return truncated or original text+ */+ private static String limitSnippet(String text, int maxChars) {+ if (text == null) return "";+ if (maxChars <= 0 || text.length() <= maxChars) return text;+ return text.substring(0, maxChars);+ }++ /**+ * Returns first non-null string from a field value that may be scalar or+ * collection.+ *+ * @param value field value+ * @return first string representation or empty string+ */+ private static String firstFieldString(Object value) {+ if (value == null) return "";+ if (value instanceof Collection) {+ for (Object item : (Collection<?>) value) if (item != null) return item.toString();+ return "";+ }+ return value.toString();+ }++ /**+ * Waits for global search feeding completion and then resorts cached results.+ *+ * @param search active search event+ * @param timeoutMs max wait time+ */+ private static void waitForFeedingAndResort(SearchEvent search, long timeoutMs) {+ if (search == null || timeoutMs <= 0) return;+ final long end = System.currentTimeMillis() + timeoutMs;+ while (!search.isFeedingFinished() && System.currentTimeMillis() < end) {+ try {+ Thread.sleep(100);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ break;+ }+ }+ search.resortCachedResults();+ }++ /**+ * Reads a single string value from a (possibly multivalued) Solr field.+ *+ * @param doc source Solr document+ * @param field schema field descriptor+ * @return first string value or empty string+ */+ private static String getOneString(SolrDocument doc, CollectionSchema field) {+ assert field.isMultiValued();+ assert field.getType() == SolrType.string || field.getType() == SolrType.text_general;+ Object r = doc.getFieldValue(field.getSolrFieldName());+ if (r == null) return "";+ if (r instanceof ArrayList) {+ return (String) ((ArrayList<?>) r).get(0);+ }+ return r.toString();+ }++ /**+ * Represents one candidate snippet around a search result and stores its+ * relevance score relative to the query.+ */+ private static class Snippet {+ private String text, url, title;+ private double score;++ /**+ * Scores text chunks and keeps the best chunk plus direct neighbors to+ * preserve context continuity.+ *+ * @param query query text+ * @param text source document/snippet text+ * @param url source URL+ * @param title source title+ * @param maxChunkLength target chunk size+ */+ public Snippet(String query, String text, String url, String title, int maxChunkLength) {+ this.url = url;+ this.title = title;+ this.score = 0.0;++ if (text == null || text.isEmpty() || maxChunkLength <= 0 || query == null) {+ this.text = "";+ return;+ }++ List<String> chunks = slicer(text, maxChunkLength);+ if (chunks.isEmpty()) {+ this.text = "";+ return;+ }+ List<String> chunksLowerCase = new ArrayList<>(chunks.size());+ // Cache lowercase chunks so token comparisons are case-insensitive.+ for (String chunk: chunks) chunksLowerCase.add(chunk.toLowerCase());++ Set<String> queryWordSet = querySet(query);+ if (queryWordSet.isEmpty()) {+ this.text = "";+ return;+ }++ int totalChunks = chunksLowerCase.size();+ Map<String, Double> idf = new HashMap<>();+ for (String word: queryWordSet) {+ int docFreq = 0;+ for (String chunk: chunksLowerCase) {+ if (chunk.contains(word)) docFreq++;+ }+ // Smoothed IDF to avoid divide-by-zero and extreme values.+ idf.put(word, Math.log((double) totalChunks / (docFreq + 1)) + 1);+ }++ Map<Integer, Double> chunkScores = new HashMap<>();+ for (int i = 0; i < chunksLowerCase.size(); i++) {+ String chunk = chunksLowerCase.get(i);+ double score = 0.0;+ Map<String, Integer> tf = new HashMap<>();++ String[] wordsInChunk = chunk.split("\\s+");+ for (String w : wordsInChunk) {+ String cleanWord = w.replaceAll("[.,!?;:]", "");+ if (cleanWord.length() > 0 && queryWordSet.contains(cleanWord)) {+ tf.put(cleanWord, tf.getOrDefault(cleanWord, 0) + 1);+ }+ }++ for (String word: queryWordSet) {+ int tfValue = tf.getOrDefault(word, 0);+ double tfIdf = (double) tfValue * idf.getOrDefault(word, 1.0);+ score += tfIdf;+ }+ chunkScores.put(i, score);+ }++ int topChunkIndex = -1;+ for (Map.Entry<Integer, Double> entry: chunkScores.entrySet()) {+ // Keep best-scoring chunk index.+ if (entry.getValue() > this.score) {+ this.score = entry.getValue();+ topChunkIndex = entry.getKey();+ }+ }++ if (topChunkIndex < 0) {+ this.text = "";+ this.score = 0.0;+ return;+ }++ List<String> snippetChunks = new ArrayList<>();+ // Include neighboring chunks to reduce abrupt starts/ends.+ if (topChunkIndex > 0) snippetChunks.add(chunks.get(topChunkIndex - 1));+ snippetChunks.add(chunks.get(topChunkIndex));+ if (topChunkIndex < chunks.size() - 1) snippetChunks.add(chunks.get(topChunkIndex + 1));+ this.text = String.join(" ", snippetChunks);+ }++ public double getScore() { return this.score; }+ public String getText() { return this.text; }+ public String getURL() { return this.url; }+ public String getTitle() { return this.title; }+ }+}diff --git a/source/net/yacy/ai/ToolCallProtocol.java b/source/net/yacy/ai/ToolCallProtocol.javanew file mode 100644index 000000000..8f526beaa--- /dev/null+++ b/source/net/yacy/ai/ToolCallProtocol.java@@ -0,0 +1,634 @@+/**+ * ToolCallProtocol+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai;++import java.io.BufferedReader;+import java.io.IOException;+import java.io.InputStreamReader;+import java.io.OutputStream;+import java.net.HttpURLConnection;+import java.net.URI;+import java.net.URISyntaxException;+import java.net.URL;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Collections;+import java.util.HashMap;+import java.util.List;+import java.util.Map;++import javax.servlet.ServletOutputStream;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;+import org.json.JSONTokener;+++/**+ * Implements the protocol glue for streamed chat completions that include tool+ * calls.+ * <p>+ * Main responsibilities:+ * <ul>+ * <li>Collect partial tool-call fragments from streaming deltas.</li>+ * <li>Merge fragments into complete tool calls (index-based merge).</li>+ * <li>Execute tools once a tool turn is complete.</li>+ * <li>Append assistant + tool messages back to the conversation.</li>+ * <li>Start follow-up model rounds until no further tool call is requested.</li>+ * </ul>+ * <p>+ * The class also enriches streamed lines with {@code tool-calls} and+ * {@code tool-results} metadata so clients can display tool activity while the+ * stream is forwarded.+ */+public final class ToolCallProtocol {++ /**+ * Hard stop to prevent infinite tool loops (assistant requests tool calls+ * repeatedly without converging to a final answer).+ */+ private static final int MAX_TOOL_ROUNDS = 8;++ /**+ * Utility class; not instantiable.+ */+ private ToolCallProtocol() {}++ /**+ * Builds a request body that is ready for tool-aware chat completion calls.+ * <p>+ * This method clones the input object, injects tool definitions and optionally+ * enforces streaming.+ *+ * @param body original request body+ * @param forceStream when true, sets {@code stream=true} in the cloned body+ * @return prepared request body clone+ */+ public static JSONObject prepareToolRequestBody(JSONObject body, boolean forceStream) {+ try {+ final JSONObject prepared = body == null ? new JSONObject(true) : new JSONObject(body.toString());+ if (forceStream) prepared.put("stream", true);+ net.yacy.ai.ToolProvider.ensureTools(prepared);+ return prepared;+ } catch (JSONException e) {+ final JSONObject fallback = new JSONObject(true);+ net.yacy.ai.ToolProvider.ensureTools(fallback);+ return fallback;+ }+ }++ /**+ * Runs the full server-side tool-stream lifecycle for one chat request.+ * <p>+ * Lifecycle:+ * <ol>+ * <li>Prepare request body for tools.</li>+ * <li>Send initial upstream chat request.</li>+ * <li>Forward initial stream while collecting tool calls.</li>+ * <li>If tool calls appeared, continue tool rounds until completion.</li>+ * </ol>+ *+ * @param out downstream SSE output stream+ * @param llm4Chat model config+ * @param originalBody request body template+ * @param messages mutable message history+ * @param initialMetadata optional JSON fields to inject into the first JSON data line+ * @return upstream HTTP status code for the initial request+ * @throws IOException on network/stream/protocol errors+ */+ public static int proxyToolLifecycle(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata) throws IOException {+ final JSONObject preparedBody = prepareToolRequestBody(originalBody, false);+ final HttpURLConnection conn = openChatCompletionConnection(llm4Chat, preparedBody);+ final int status = conn.getResponseCode();+ if (status == 200) {+ handleInitialStreamAndContinue(out, conn, llm4Chat, preparedBody, messages, initialMetadata);+ }+ return status;+ }++ /**+ * Extracts streamed assistant content and tool-call deltas from one chat chunk.+ * <p>+ * Supports both modern {@code delta.tool_calls} and legacy+ * {@code delta.function_call} formats.+ *+ * @param j streamed JSON chunk+ * @param toolCalls mutable map collecting tool calls by index+ * @param assistantContent buffer accumulating assistant text tokens+ * @param sawToolCalls single-item boolean holder set to true once any tool+ * call appears in this stream+ */+ public static void captureToolCalls(JSONObject j, Map<Integer, ToolCall> toolCalls, StringBuilder assistantContent, boolean[] sawToolCalls) {+ if (j == null) return;+ // Only first choice is processed because streaming APIs typically emit one choice.+ JSONArray choices = j.optJSONArray("choices");+ if (choices == null || choices.length() == 0) return;+ JSONObject choice = choices.optJSONObject(0);+ if (choice == null) return;+ JSONObject delta = choice.optJSONObject("delta");+ if (delta == null) return;++ // Assistant text arrives token-by-token in streaming mode.+ String content = delta.optString("content", null);+ if (content != null) assistantContent.append(content);++ // Preferred modern tool-call format.+ JSONArray toolDelta = delta.optJSONArray("tool_calls");+ if (toolDelta != null) {+ sawToolCalls[0] = true;+ mergeToolCalls(toolCalls, toolDelta);+ }++ // Legacy compatibility: transform function_call into tool_calls-like shape.+ JSONObject functionCall = delta.optJSONObject("function_call");+ if (functionCall != null) {+ sawToolCalls[0] = true;+ try {+ JSONArray wrapper = new JSONArray();+ JSONObject entry = new JSONObject(true);+ entry.put("index", 0);+ entry.put("type", "function");+ entry.put("function", functionCall);+ wrapper.put(entry);+ mergeToolCalls(toolCalls, wrapper);+ } catch (JSONException e) {+ // ignore malformed legacy function_call chunk+ }+ }+ }++ /**+ * Forwards the initial upstream stream to the client while collecting tool+ * calls, then executes follow-up tool rounds when needed.+ *+ * @param out downstream SSE output stream+ * @param conn initial upstream connection+ * @param llm4Chat model config for follow-up rounds+ * @param originalBody request body template for follow-up rounds+ * @param messages mutable conversation history+ * @param initialMetadata optional metadata injected once into the first JSON line+ * @throws IOException on I/O errors+ */+ private static void handleInitialStreamAndContinue(ServletOutputStream out, HttpURLConnection conn, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata) throws IOException {+ final StringBuilder assistantContent = new StringBuilder();+ final Map<Integer, ToolCall> toolCalls = new HashMap<>();+ final boolean[] sawToolCalls = new boolean[]{false};+ boolean firstJsonDataLinePending = initialMetadata != null && initialMetadata.length() > 0;++ try (BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) {+ String line;+ while ((line = in.readLine()) != null) {+ boolean isDoneLine = "data: [DONE]".equals(line.trim()) || "[DONE]".equals(line.trim());+ if (firstJsonDataLinePending) {+ String patched = injectJsonMetadata(line, initialMetadata);+ if (patched != null) {+ line = patched;+ firstJsonDataLinePending = false;+ }+ }+ if (line.startsWith("data:")) {+ int p = line.indexOf('{');+ if (p > 0) {+ try {+ JSONObject j = new JSONObject(new JSONTokener(line.substring(p)));+ captureToolCalls(j, toolCalls, assistantContent, sawToolCalls);+ } catch (JSONException e) {+ // ignore malformed stream chunks and continue forwarding stream+ }+ }+ }+ if (!isDoneLine || !sawToolCalls[0]) {+ out.println(line);+ out.flush();+ }+ }+ } finally {+ conn.disconnect();+ }++ if (sawToolCalls[0]) {+ handleToolCallsAndContinue(out, llm4Chat, originalBody, messages, assistantContent.toString(), toolCalls);+ }+ }++ /**+ * Runs the complete tool-turn loop after initial stream parsing detected tool+ * calls.+ * <p>+ * Per round:+ * <ol>+ * <li>Append assistant tool request + tool response messages.</li>+ * <li>Call the chat completion endpoint again with updated messages.</li>+ * <li>Forward streamed lines to the client while capturing potential next+ * tool calls.</li>+ * <li>Stop when no more tool calls are requested or the round cap is hit.</li>+ * </ol>+ *+ * @param out output stream to the client (SSE style lines)+ * @param llm4Chat selected model configuration+ * @param originalBody original request body template+ * @param messages mutable conversation messages from original request+ * @param assistantContent already streamed assistant text from current round+ * @param toolCalls collected tool calls from current round+ * @throws IOException when network I/O or JSON processing fails+ */+ public static void handleToolCallsAndContinue(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls) throws IOException {+ try {+ // Work on a copy so caller-owned message arrays are not modified unexpectedly.+ JSONArray newMessages = new JSONArray();+ for (int i = 0; i < messages.length(); i++) {+ newMessages.put(messages.get(i));+ }+ String roundAssistantContent = assistantContent == null ? "" : assistantContent;+ Map<Integer, ToolCall> roundToolCalls = new HashMap<>(toolCalls);+ // Track total calls per tool across this whole tool-turn lifecycle.+ Map<String, Integer> toolCallCounters = new HashMap<>();++ for (int round = 0; round < MAX_TOOL_ROUNDS; round++) {+ // Convert captured tool calls into assistant/tool messages and execute tools.+ ToolRoundData roundData = appendAssistantAndToolMessages(newMessages, roundAssistantContent, roundToolCalls, toolCallCounters);+ if (roundData == null || roundData.toolResults.length() == 0) {+ // No executable tool calls; terminate stream cleanly.+ out.println("data: [DONE]");+ out.flush();+ return;+ }++ // Build follow-up completion request from original body template.+ final JSONObject followup = prepareToolRequestBody(originalBody, true);+ followup.put("messages", newMessages);+ final HttpURLConnection followConn = openChatCompletionConnection(llm4Chat, followup);+ if (followConn.getResponseCode() != 200) {+ // Upstream error: close downstream stream instead of sending broken chunks.+ out.println("data: [DONE]");+ out.flush();+ followConn.disconnect();+ return;+ }++ // Collect data for the next potential tool round while forwarding this stream.+ final StringBuilder nextAssistantContent = new StringBuilder();+ final Map<Integer, ToolCall> nextToolCalls = new HashMap<>();+ final boolean[] sawToolCalls = new boolean[]{false};++ try (BufferedReader in = new BufferedReader(new InputStreamReader(followConn.getInputStream(), StandardCharsets.UTF_8))) {+ String line;+ boolean injectedToolResults = false;+ while ((line = in.readLine()) != null) {+ // Inject tool metadata once into outgoing stream so UI can show executed tools.+ if (!injectedToolResults && roundData.toolResults.length() > 0) {+ line = injectToolMetadata(line, roundData);+ if (line != null && line.indexOf("\"tool-results\"") >= 0) {+ injectedToolResults = true;+ }+ }+ boolean isDoneLine = "data: [DONE]".equals(line.trim()) || "[DONE]".equals(line.trim());+ if (line.startsWith("data:")) {+ int p = line.indexOf('{');+ if (p > 0) {+ try {+ JSONObject j = new JSONObject(new JSONTokener(line.substring(p)));+ // Parse streamed deltas to capture possible next tool round.+ captureToolCalls(j, nextToolCalls, nextAssistantContent, sawToolCalls);+ } catch (JSONException e) {+ // ignore malformed stream chunks and continue forwarding stream+ }+ }+ }+ // Hide interim [DONE] if another tool round is expected.+ if (!isDoneLine || !sawToolCalls[0]) {+ out.println(line);+ out.flush();+ }+ }+ } finally {+ followConn.disconnect();+ }++ // Ignore phantom rounds where stream signaled tool-calls but no concrete calls were parsed.+ if (nextToolCalls.isEmpty()) sawToolCalls[0] = false;+ // Final answer reached; caller already received forwarded stream lines.+ if (!sawToolCalls[0]) return;++ // Continue with next round tool request that was found in follow-up stream.+ roundAssistantContent = nextAssistantContent.toString();+ roundToolCalls = nextToolCalls;+ }+ // Safety fallback when round cap is hit.+ out.println("data: [DONE]");+ out.flush();+ } catch (JSONException e) {+ throw new IOException("JSON processing error in tool handling", e);+ }+ }++ /**+ * Opens and sends one chat completion request.+ *+ * @param llm4Chat model config used for endpoint and auth+ * @param requestBody request payload+ * @return opened connection after request body has been sent+ * @throws IOException on URI/network/write failures+ */+ private static HttpURLConnection openChatCompletionConnection(LLM.LLMModel llm4Chat, JSONObject requestBody) throws IOException {+ final URL url;+ try {+ url = new URI(llm4Chat.llm.hoststub + "/v1/chat/completions").toURL();+ } catch (URISyntaxException e) {+ throw new IOException("Invalid chat completion URL: " + e.getMessage(), e);+ }+ final HttpURLConnection conn = (HttpURLConnection) url.openConnection();+ conn.setRequestMethod("POST");+ conn.setRequestProperty("Content-Type", "application/json");+ if (!llm4Chat.llm.api_key.isEmpty()) {+ conn.setRequestProperty("Authorization", "Bearer " + llm4Chat.llm.api_key);+ }+ conn.setDoOutput(true);+ try (OutputStream os = conn.getOutputStream()) {+ os.write(requestBody.toString().getBytes(StandardCharsets.UTF_8));+ os.flush();+ }+ return conn;+ }++ /**+ * Merges incremental tool call fragments from a streaming delta into the+ * in-progress tool call map.+ * <p>+ * Streaming responses may deliver one tool call over multiple chunks. This method+ * merges fields by tool-call index:+ * <ul>+ * <li>{@code id}, {@code type}, and function {@code name} keep the latest non-empty value.</li>+ * <li>function {@code arguments} are appended because they are often streamed in pieces.</li>+ * </ul>+ *+ * @param toolCalls current merged tool calls, keyed by call index+ * @param deltaCalls new delta chunk containing partial tool call objects+ */+ private static void mergeToolCalls(Map<Integer, ToolCall> toolCalls, JSONArray deltaCalls) {+ // Each array element can contain only a fragment of the final tool call.+ for (int i = 0; i < deltaCalls.length(); i++) {+ JSONObject delta = deltaCalls.optJSONObject(i);+ if (delta == null) continue;+ Integer index = readToolCallIndex(delta);+ // Ignore chunks without a valid non-negative call index.+ if (index == null || index.intValue() < 0) continue;++ // Reuse existing partial state for this index or start a new one.+ ToolCall call = toolCalls.getOrDefault(index, new ToolCall());++ // Scalar fields are replaced when a newer non-empty value arrives.+ String id = delta.optString("id", null);+ if (id != null && !id.isEmpty()) call.id = id;+ String type = delta.optString("type", null);+ if (type != null && !type.isEmpty()) call.type = type;+ JSONObject fn = delta.optJSONObject("function");+ if (fn != null) {+ String name = fn.optString("name", null);+ if (name != null && !name.isEmpty()) call.name = name;+ Object rawArgs = fn.opt("arguments");+ if (rawArgs instanceof String) {+ String args = (String) rawArgs;+ // Arguments are streamed as string fragments and must be concatenated.+ if (!args.isEmpty()) call.arguments = (call.arguments == null ? "" : call.arguments) + args;+ }+ }+ toolCalls.put(index, call);+ }+ }++ /**+ * Appends one assistant message containing tool_calls plus corresponding tool+ * role messages with execution results.+ * <p>+ * Only executable tool calls are included (must have a function name and valid+ * JSON arguments if arguments are present).+ *+ * @param messages mutable conversation message array to append to+ * @param assistantContent assistant text that accompanied tool call emission+ * @param toolCalls merged tool calls keyed by index+ * @param toolCallCounters cumulative per-tool call counters for this turn+ * @return round data containing emitted {@code tool_calls} and tool execution+ * results, or {@code null} when JSON assembly fails+ */+ private static ToolRoundData appendAssistantAndToolMessages(JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls, Map<String, Integer> toolCallCounters) {+ try {+ // Keep original model call order stable by sorting call indices.+ List<Integer> indices = new ArrayList<>(toolCalls.keySet());+ Collections.sort(indices);+ JSONArray toolCallsArray = new JSONArray();+ JSONArray toolMessages = new JSONArray();+ JSONArray toolResults = new JSONArray();+ for (Integer idx : indices) {+ ToolCall call = toolCalls.get(idx);+ if (call == null) continue;+ // Ensure required defaults for follow-up protocol compliance.+ if (call.id == null || call.id.isEmpty()) call.id = "toolcall_" + idx + "_" + System.currentTimeMillis();+ if (call.type == null || call.type.isEmpty()) call.type = "function";+ if (!isExecutableToolCall(call)) continue;+ // Enforce per-tool max calls for the complete tool turn lifecycle.+ final String toolName = call.name == null ? "" : call.name.trim();+ final int maxCalls = net.yacy.ai.ToolProvider.maxCallsPerTurn(toolName);+ final int usedCalls = toolCallCounters.getOrDefault(toolName, Integer.valueOf(0)).intValue();+ if (usedCalls >= maxCalls) continue;++ // Assistant message representation of requested tool call.+ JSONObject toolCallJson = new JSONObject(true);+ toolCallJson.put("id", call.id);+ toolCallJson.put("type", call.type);+ JSONObject fn = new JSONObject(true);+ fn.put("name", call.name == null ? "" : call.name);+ fn.put("arguments", call.arguments == null ? "" : call.arguments);+ toolCallJson.put("function", fn);+ toolCallsArray.put(toolCallJson);++ // Execute tool locally and create "tool" role response message.+ String result = net.yacy.ai.ToolProvider.executeTool(call.name, call.arguments);+ JSONObject toolMessage = new JSONObject(true);+ toolMessage.put("role", "tool");+ toolMessage.put("tool_call_id", call.id);+ toolMessage.put("name", call.name == null ? "" : call.name);+ toolMessage.put("content", result);+ toolMessages.put(toolMessage);+ toolCallCounters.put(toolName, Integer.valueOf(usedCalls + 1));++ // Separate compact result list for stream metadata injection.+ JSONObject toolResult = new JSONObject(true);+ toolResult.put("tool_call_id", call.id);+ toolResult.put("name", call.name == null ? "" : call.name);+ toolResult.put("content", result);+ toolResults.put(toolResult);+ }++ // If nothing is executable, leave the message history unchanged.+ if (toolCallsArray.length() == 0) {+ return new ToolRoundData(new JSONArray(), new JSONArray());+ }++ // Add assistant turn that requested tools.+ JSONObject assistantMessage = new JSONObject(true);+ assistantMessage.put("role", "assistant");+ assistantMessage.put("content", assistantContent == null ? "" : assistantContent);+ assistantMessage.put("tool_calls", toolCallsArray);+ messages.put(assistantMessage);+ // Follow with synthetic tool result messages for next model round context.+ for (int i = 0; i < toolMessages.length(); i++) messages.put(toolMessages.get(i));++ return new ToolRoundData(toolCallsArray, toolResults);+ } catch (JSONException e) {+ return null;+ }+ }++ /**+ * Injects tool metadata into one outbound SSE line if the line contains JSON+ * data.+ * <p>+ * Metadata keys:+ * <ul>+ * <li>{@code tool-calls}: normalized tool call objects sent in the round.</li>+ * <li>{@code tool-results}: local tool execution outputs.</li>+ * </ul>+ *+ * @param line outbound line (typically prefixed with {@code data:})+ * @param roundData round metadata to inject+ * @return original or augmented line+ */+ private static String injectToolMetadata(String line, ToolRoundData roundData) {+ if (line == null || roundData == null || roundData.toolResults == null || roundData.toolResults.length() == 0) return line;+ if (!line.startsWith("data:")) return line;+ int p = line.indexOf('{');+ if (p <= 0) return line;+ try {+ JSONObject j = new JSONObject(new JSONTokener(line.substring(p)));+ // Do not overwrite if upstream already provided these fields.+ if (!j.has("tool-calls") && roundData.toolCalls != null) j.put("tool-calls", roundData.toolCalls);+ if (!j.has("tool-results")) j.put("tool-results", roundData.toolResults);+ return line.substring(0, p) + j.toString();+ } catch (JSONException e) {+ // Non-JSON data lines are forwarded unchanged.+ return line;+ }+ }++ /**+ * Injects arbitrary JSON metadata into one SSE {@code data: ...} JSON line.+ * Existing keys are preserved.+ *+ * @param line outbound stream line+ * @param metadata metadata object to inject+ * @return augmented line, or {@code null} when the line is not a JSON data line+ */+ private static String injectJsonMetadata(String line, JSONObject metadata) {+ if (line == null || metadata == null || metadata.length() == 0) return null;+ if (!line.startsWith("data:")) return null;+ int p = line.indexOf('{');+ if (p <= 0) return null;+ try {+ JSONObject j = new JSONObject(new JSONTokener(line.substring(p)));+ JSONArray names = metadata.names();+ if (names == null || names.length() == 0) return null;+ for (int i = 0; i < names.length(); i++) {+ String name = names.optString(i, null);+ if (name == null || name.isEmpty()) continue;+ if (!j.has(name)) j.put(name, metadata.get(name));+ }+ return line.substring(0, p) + j.toString();+ } catch (JSONException e) {+ return null;+ }+ }++ /**+ * Reads the {@code index} field used to correlate streamed tool-call fragments.+ *+ * @param delta one tool-call delta object+ * @return index as Integer when present and numeric, otherwise {@code null}+ */+ private static Integer readToolCallIndex(JSONObject delta) {+ if (delta == null || !delta.has("index")) return null;+ Object raw = delta.opt("index");+ if (raw instanceof Number) return Integer.valueOf(((Number) raw).intValue());+ return null;+ }++ /**+ * Checks whether a merged tool call is executable locally.+ * <p>+ * Requirements:+ * <ul>+ * <li>non-empty tool/function name</li>+ * <li>arguments omitted/empty or valid JSON object text</li>+ * </ul>+ *+ * @param call merged tool call candidate+ * @return true when tool execution should be attempted+ */+ private static boolean isExecutableToolCall(ToolCall call) {+ if (call == null || call.name == null || call.name.trim().isEmpty()) return false;+ String args = call.arguments;+ if (args == null || args.isEmpty()) return true;+ try {+ new JSONObject(args);+ return true;+ } catch (JSONException e) {+ return false;+ }+ }++ /**+ * Round-local metadata container used during tool loop execution.+ */+ private static final class ToolRoundData {+ /** Tool calls that were executed in the round. */+ final JSONArray toolCalls;+ /** Tool outputs corresponding to {@link #toolCalls}. */+ final JSONArray toolResults;++ /**+ * Builds a round data object with non-null arrays.+ *+ * @param toolCalls tool call descriptors+ * @param toolResults tool execution outputs+ */+ ToolRoundData(JSONArray toolCalls, JSONArray toolResults) {+ this.toolCalls = toolCalls == null ? new JSONArray() : toolCalls;+ this.toolResults = toolResults == null ? new JSONArray() : toolResults;+ }+ }++ /**+ * Mutable tool-call assembly object built from streamed deltas.+ * <p>+ * Fields may be populated incrementally over multiple chunks.+ */+ public static final class ToolCall {+ /** Tool call id as emitted by model/provider. */+ public String id;+ /** Tool call type, usually {@code function}. */+ public String type;+ /** Tool/function name. */+ public String name;+ /** JSON argument text (can be assembled from fragments). */+ public String arguments;+ }+}diff --git a/source/net/yacy/ai/ToolHandler.java b/source/net/yacy/ai/ToolHandler.javanew file mode 100644index 000000000..9c278cf00--- /dev/null+++ b/source/net/yacy/ai/ToolHandler.java@@ -0,0 +1,43 @@+/**+ * ToolHandler+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai;++import org.json.JSONException;+import org.json.JSONObject;++public interface ToolHandler {++ JSONObject definition() throws JSONException;++ String execute(String arguments);++ int maxCallsPerTurn();++ public static String errorJson(String message) {+ try {+ JSONObject err = new JSONObject(true);+ err.put("error", message == null ? "error" : message);+ return err.toString();+ } catch (JSONException e) {+ return "{\"error\":\"" + (message == null ? "error" : message.replace("\"", "'")) + "\"}";+ }+ }+}diff --git a/source/net/yacy/ai/ToolProvider.java b/source/net/yacy/ai/ToolProvider.javanew file mode 100644index 000000000..070c6dd45--- /dev/null+++ b/source/net/yacy/ai/ToolProvider.java@@ -0,0 +1,226 @@+/**+ * ToolProvider+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai;++import java.util.Arrays;+import java.util.Collections;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.ai.tools.CalculatorTool;+import net.yacy.ai.tools.ChitChatTool;+import net.yacy.ai.tools.DateMathTool;+import net.yacy.ai.tools.DateTimeTool;+import net.yacy.ai.tools.HttpJsonTool;+import net.yacy.ai.tools.NumberParserTool;+import net.yacy.ai.tools.PromptToMermaidTool;+import net.yacy.ai.tools.SelfReflectTool;+import net.yacy.ai.tools.TableOpsTool;+import net.yacy.ai.tools.UnitConverterTool;+import net.yacy.ai.tools.WebFetchTool;+import net.yacy.ai.tools.WikipediaLinkCreatorTool;++/**+ * Central registry and dispatch utility for all built-in YaCy LLM tools.+ * <p>+ * Responsibilities:+ * <ul>+ * <li>Expose tool JSON schemas into outbound chat-completion requests.</li>+ * <li>Resolve tool names to handlers.</li>+ * <li>Execute tool calls by name with raw JSON argument text.</li>+ * </ul>+ */+public final class ToolProvider {++ /**+ * Ordered list of built-in tool handlers. Order is preserved when exposing+ * definitions to providers.+ */+ private static final List<ToolHandler> TOOLS = Arrays.asList(+ new DateTimeTool(),+ new DateMathTool(),+ new CalculatorTool(),+ new NumberParserTool(),+ new UnitConverterTool(),+ new WebFetchTool(),+ new HttpJsonTool(),+ new TableOpsTool(),+ new SelfReflectTool(),+ new ChitChatTool(),+ new PromptToMermaidTool(),+ new WikipediaLinkCreatorTool()+ );++ /**+ * Lookup table for fast runtime dispatch by function/tool name.+ */+ private static final Map<String, ToolHandler> TOOL_BY_NAME = buildToolIndex(TOOLS);++ /**+ * Utility class; not instantiable.+ */+ private ToolProvider() {}++ /**+ * Ensures a chat request body contains all available tool definitions and a+ * default tool selection mode.+ * <p>+ * Existing tool definitions are kept; missing ones are appended.+ *+ * @param body request body to mutate+ */+ public static void ensureTools(JSONObject body) {+ if (body == null) return;+ try {+ // Create "tools" array lazily when caller did not provide one.+ JSONArray tools = body.optJSONArray("tools");+ if (tools == null) {+ tools = new JSONArray();+ body.put("tools", tools);+ }+ // Merge registry definitions without duplicating by tool name.+ for (ToolHandler tool : TOOLS) {+ JSONObject definition = tool.definition();+ addToolDefinitionIfMissing(tools, definition);+ }+ // Providers usually expect explicit tool-choice mode.+ if (!body.has("tool_choice")) {+ body.put("tool_choice", "auto");+ }+ } catch (JSONException e) {+ // keep body unchanged if tool schema cannot be injected+ }+ }++ /**+ * Dispatches one tool invocation by name.+ *+ * @param name tool/function name+ * @param arguments raw JSON arguments string+ * @return tool execution result as JSON string (or error JSON)+ */+ public static String executeTool(String name, String arguments) {+ if (name == null) return errorJson("Invalid tool call");+ ToolHandler tool = TOOL_BY_NAME.get(name);+ if (tool == null) return errorJson("Unknown tool: " + name);+ // Tool implementations are responsible for argument parsing/validation.+ return tool.execute(arguments);+ }++ /**+ * Returns the configured maximum number of calls for a tool within one tool+ * turn lifecycle.+ *+ * @param name tool/function name+ * @return maximum call count, defaults to 1 for unknown/invalid values+ */+ public static int maxCallsPerTurn(String name) {+ if (name == null) return 1;+ ToolHandler tool = TOOL_BY_NAME.get(name);+ if (tool == null) return 1;+ int max = tool.maxCallsPerTurn();+ return max <= 0 ? 1 : max;+ }++ /**+ * Builds an immutable map from tool name to handler.+ * Invalid or unnamed definitions are skipped.+ *+ * @param tools source handlers+ * @return unmodifiable name-indexed map+ */+ private static Map<String, ToolHandler> buildToolIndex(List<ToolHandler> tools) {+ Map<String, ToolHandler> byName = new LinkedHashMap<>();+ for (ToolHandler tool : tools) {+ if (tool == null) continue;+ try {+ String name = extractToolName(tool.definition());+ if (name == null || name.isEmpty()) continue;+ byName.put(name, tool);+ } catch (JSONException e) {+ // skip invalid tool definition+ }+ }+ return Collections.unmodifiableMap(byName);+ }++ /**+ * Appends a tool definition if no existing tool with the same function name+ * is already present.+ *+ * @param tools destination tool array in request body+ * @param toolDefinition candidate definition+ */+ private static void addToolDefinitionIfMissing(JSONArray tools, JSONObject toolDefinition) {+ if (tools == null || toolDefinition == null) return;+ try {+ String name = extractToolName(toolDefinition);+ if (name == null || name.isEmpty()) return;+ // Dedupe by logical function name (not by full JSON string equality).+ for (int i = 0; i < tools.length(); i++) {+ JSONObject existing = tools.optJSONObject(i);+ if (existing == null) continue;+ String existingName = extractToolName(existing);+ if (name.equals(existingName)) return;+ }+ tools.put(toolDefinition);+ } catch (JSONException e) {+ // ignore malformed tool schema+ }+ }++ /**+ * Extracts the function name from a tool definition object.+ *+ * @param toolDefinition MCP/OpenAI-style tool definition+ * @return trimmed function name or {@code null}+ * @throws JSONException if JSON access fails unexpectedly+ */+ private static String extractToolName(JSONObject toolDefinition) throws JSONException {+ if (toolDefinition == null) return null;+ JSONObject fn = toolDefinition.optJSONObject("function");+ if (fn == null) return null;+ String name = fn.optString("name", "");+ return name == null ? null : name.trim();+ }++ /**+ * Builds a minimal error JSON string while avoiding propagation of secondary+ * JSON construction failures.+ *+ * @param message human-readable error message+ * @return serialized JSON error object+ */+ private static String errorJson(String message) {+ try {+ JSONObject err = new JSONObject(true);+ err.put("error", message == null ? "error" : message);+ return err.toString();+ } catch (JSONException e) {+ return "{\"error\":\"" + (message == null ? "error" : message.replace("\"", "'")) + "\"}";+ }+ }+}diff --git a/source/net/yacy/ai/tools/CalculatorTool.java b/source/net/yacy/ai/tools/CalculatorTool.javanew file mode 100644index 000000000..4c33d7194--- /dev/null+++ b/source/net/yacy/ai/tools/CalculatorTool.java@@ -0,0 +1,358 @@+/**+ * CalculatorTool+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.math.BigDecimal;+import java.math.MathContext;+import java.util.ArrayList;+import java.util.List;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.ai.ToolHandler;++public class CalculatorTool implements ToolHandler {++ private static final String NAME = "calculator";++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "Evaluate a mathematical formula. Supports operators (+,-,*,/,%,^), constants (pi,e,tau,phi), functions (sqrt,abs,ln,log,sin,cos,tan,asin,acos,atan,min,max,pow,root,...) and scientific notation. Use this as your calculator.");+ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ JSONObject props = new JSONObject(true);+ JSONObject formula = new JSONObject(true);+ formula.put("type", "string");+ formula.put("description", "Formula to evaluate, e.g. (2+3)*4/5");+ props.put("formula", formula);+ params.put("properties", props);+ params.put("required", new JSONArray().put("formula"));+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }+++ public int maxCallsPerTurn() {+ return 10;+ }+++ public String execute(String arguments) {+ String formula;+ try {+ JSONObject obj = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);+ formula = obj.optString("formula", "").trim();+ } catch (JSONException e) {+ return ToolHandler.errorJson("Invalid arguments JSON");+ }+ if (formula == null || formula.isEmpty()) return ToolHandler.errorJson("Missing formula");+ try {+ BigDecimal value = new ExpressionParser(formula).parse();+ JSONObject result = new JSONObject(true);+ result.put("formula", formula);+ result.put("result", value.stripTrailingZeros().toPlainString());+ return result.toString();+ } catch (Exception e) {+ return ToolHandler.errorJson("Invalid formula: " + e.getMessage());+ }+ }++ private static final class ExpressionParser {+ private final String expression;+ private int pos;++ private ExpressionParser(String expression) {+ this.expression = expression == null ? "" : expression;+ }++ private BigDecimal parse() {+ BigDecimal value = parseExpression();+ skipWs();+ if (pos != expression.length()) throw new IllegalArgumentException("Unexpected token at " + pos);+ return value;+ }++ private BigDecimal parseExpression() {+ BigDecimal value = parseTerm();+ while (true) {+ skipWs();+ if (eat('+')) value = value.add(parseTerm());+ else if (eat('-')) value = value.subtract(parseTerm());+ else return value;+ }+ }++ private BigDecimal parseTerm() {+ BigDecimal value = parseFactor();+ while (true) {+ skipWs();+ if (eat('*')) value = value.multiply(parseFactor(), MathContext.DECIMAL64);+ else if (eat('/')) value = value.divide(parseFactor(), MathContext.DECIMAL64);+ else if (eat('%')) value = value.remainder(parseFactor(), MathContext.DECIMAL64);+ else return value;+ }+ }++ private BigDecimal parseFactor() {+ skipWs();+ if (eat('+')) return parseFactor();+ if (eat('-')) return parseFactor().negate();+ BigDecimal base;+ if (eat('(')) {+ base = parseExpression();+ if (!eat(')')) throw new IllegalArgumentException("Missing ')'");+ } else if (isIdentifierStart(currentChar())) {+ String ident = parseIdentifier();+ if (eat('(')) {+ List<BigDecimal> args = parseArguments();+ if (!eat(')')) throw new IllegalArgumentException("Missing ')' after function argument");+ base = applyFunction(ident, args);+ } else {+ base = resolveConstant(ident);+ }+ } else {+ base = parseNumber();+ }+ skipWs();+ if (eat('^')) {+ BigDecimal exp = parseFactor();+ try {+ int intExp = exp.intValueExact();+ return base.pow(intExp, MathContext.DECIMAL64);+ } catch (ArithmeticException e) {+ double b = base.doubleValue();+ double ex = exp.doubleValue();+ // Fractional powers are evaluated in double precision.+ double result = Math.pow(b, ex);+ if (Double.isNaN(result) || Double.isInfinite(result)) {+ throw new IllegalArgumentException("Invalid exponentiation result");+ }+ return new BigDecimal(result, MathContext.DECIMAL64);+ }+ }+ return base;+ }++ private BigDecimal parseNumber() {+ skipWs();+ int start = pos;+ boolean dot = false;+ boolean exponent = false;+ while (pos < expression.length()) {+ char c = expression.charAt(pos);+ if (c >= '0' && c <= '9') {+ pos++;+ continue;+ }+ if (c == '.' && !dot) {+ dot = true;+ pos++;+ continue;+ }+ if ((c == 'e' || c == 'E') && !exponent) {+ exponent = true;+ pos++;+ if (pos < expression.length()) {+ char sign = expression.charAt(pos);+ if (sign == '+' || sign == '-') pos++;+ }+ continue;+ }+ break;+ }+ if (start == pos) throw new IllegalArgumentException("Number expected at " + pos);+ return new BigDecimal(expression.substring(start, pos), MathContext.DECIMAL64);+ }++ private String parseIdentifier() {+ skipWs();+ int start = pos;+ if (!isIdentifierStart(currentChar())) throw new IllegalArgumentException("Identifier expected at " + pos);+ pos++;+ while (pos < expression.length()) {+ char c = expression.charAt(pos);+ if (Character.isLetterOrDigit(c) || c == '_' || c == '.') {+ pos++;+ continue;+ }+ break;+ }+ return expression.substring(start, pos).toLowerCase();+ }++ private List<BigDecimal> parseArguments() {+ List<BigDecimal> args = new ArrayList<>();+ skipWs();+ if (currentChar() == ')') return args;+ args.add(parseExpression());+ while (true) {+ skipWs();+ if (!eat(',')) break;+ args.add(parseExpression());+ }+ return args;+ }++ private BigDecimal applyFunction(String name, List<BigDecimal> args) {+ String fn = shortName(name);+ if ("sqrt".equals(fn)) return bd(Math.sqrt(nonNegative(arg(args, 0), "sqrt")));+ if ("cbrt".equals(fn)) return bd(Math.cbrt(arg(args, 0).doubleValue()));+ if ("abs".equals(fn)) return arg(args, 0).abs();+ if ("floor".equals(fn)) return bd(Math.floor(arg(args, 0).doubleValue()));+ if ("ceil".equals(fn)) return bd(Math.ceil(arg(args, 0).doubleValue()));+ if ("round".equals(fn)) return bd(Math.rint(arg(args, 0).doubleValue()));+ if ("trunc".equals(fn)) return bd(arg(args, 0).doubleValue() < 0d ? Math.ceil(arg(args, 0).doubleValue()) : Math.floor(arg(args, 0).doubleValue()));++ if ("exp".equals(fn)) return bd(Math.exp(arg(args, 0).doubleValue()));+ if ("ln".equals(fn)) return bd(Math.log(positive(arg(args, 0), "ln")));+ if ("log".equals(fn)) {+ if (args.size() == 1) return bd(Math.log10(positive(arg(args, 0), "log")));+ double value = positive(arg(args, 0), "log");+ double base = positive(arg(args, 1), "log base");+ if (base == 1d) throw new IllegalArgumentException("log base must not be 1");+ return bd(Math.log(value) / Math.log(base));+ }+ if ("log10".equals(fn)) return bd(Math.log10(positive(arg(args, 0), "log10")));+ if ("log2".equals(fn)) return bd(Math.log(positive(arg(args, 0), "log2")) / Math.log(2d));++ if ("sin".equals(fn)) return bd(Math.sin(arg(args, 0).doubleValue()));+ if ("cos".equals(fn)) return bd(Math.cos(arg(args, 0).doubleValue()));+ if ("tan".equals(fn)) return bd(Math.tan(arg(args, 0).doubleValue()));+ if ("asin".equals(fn)) return bd(Math.asin(arg(args, 0).doubleValue()));+ if ("acos".equals(fn)) return bd(Math.acos(arg(args, 0).doubleValue()));+ if ("atan".equals(fn)) return bd(Math.atan(arg(args, 0).doubleValue()));+ if ("atan2".equals(fn)) return bd(Math.atan2(arg(args, 0).doubleValue(), arg(args, 1).doubleValue()));+ if ("sinh".equals(fn)) return bd(Math.sinh(arg(args, 0).doubleValue()));+ if ("cosh".equals(fn)) return bd(Math.cosh(arg(args, 0).doubleValue()));+ if ("tanh".equals(fn)) return bd(Math.tanh(arg(args, 0).doubleValue()));++ if ("deg".equals(fn) || "degrees".equals(fn) || "todeg".equals(fn)) return bd(Math.toDegrees(arg(args, 0).doubleValue()));+ if ("rad".equals(fn) || "radians".equals(fn) || "torad".equals(fn)) return bd(Math.toRadians(arg(args, 0).doubleValue()));++ if ("pow".equals(fn)) return bd(Math.pow(arg(args, 0).doubleValue(), arg(args, 1).doubleValue()));+ if ("root".equals(fn)) {+ double n = arg(args, 1).doubleValue();+ if (n == 0d) throw new IllegalArgumentException("root degree must not be 0");+ return bd(Math.pow(arg(args, 0).doubleValue(), 1d / n));+ }+ if ("mod".equals(fn)) return arg(args, 0).remainder(arg(args, 1), MathContext.DECIMAL64);++ if ("min".equals(fn)) return min(args);+ if ("max".equals(fn)) return max(args);+ if ("avg".equals(fn) || "mean".equals(fn)) return mean(args);+ if ("sum".equals(fn)) return sum(args);+ throw new IllegalArgumentException("Unknown function: " + name);+ }++ private BigDecimal resolveConstant(String name) {+ String n = shortName(name);+ if ("pi".equals(n)) return new BigDecimal(Math.PI, MathContext.DECIMAL64);+ if ("e".equals(n)) return new BigDecimal(Math.E, MathContext.DECIMAL64);+ if ("tau".equals(n)) return new BigDecimal(2d * Math.PI, MathContext.DECIMAL64);+ if ("phi".equals(n)) return new BigDecimal((1d + Math.sqrt(5d)) / 2d, MathContext.DECIMAL64);+ throw new IllegalArgumentException("Unknown identifier: " + name);+ }++ private String shortName(String name) {+ if (name == null || name.isEmpty()) return "";+ int p = name.lastIndexOf('.');+ if (p < 0 || p >= name.length() - 1) return name;+ return name.substring(p + 1);+ }++ private BigDecimal arg(List<BigDecimal> args, int index) {+ if (args == null || index < 0 || index >= args.size()) throw new IllegalArgumentException("Missing function argument at position " + (index + 1));+ return args.get(index);+ }++ private double positive(BigDecimal value, String label) {+ double v = value.doubleValue();+ if (v <= 0d) throw new IllegalArgumentException(label + " argument must be > 0");+ return v;+ }++ private double nonNegative(BigDecimal value, String label) {+ double v = value.doubleValue();+ if (v < 0d) throw new IllegalArgumentException(label + " argument must be >= 0");+ return v;+ }++ private BigDecimal bd(double value) {+ if (Double.isNaN(value) || Double.isInfinite(value)) {+ throw new IllegalArgumentException("Invalid numeric result");+ }+ return new BigDecimal(value, MathContext.DECIMAL64);+ }++ private BigDecimal min(List<BigDecimal> args) {+ if (args == null || args.isEmpty()) throw new IllegalArgumentException("min requires at least one argument");+ BigDecimal m = args.get(0);+ for (int i = 1; i < args.size(); i++) if (args.get(i).compareTo(m) < 0) m = args.get(i);+ return m;+ }++ private BigDecimal max(List<BigDecimal> args) {+ if (args == null || args.isEmpty()) throw new IllegalArgumentException("max requires at least one argument");+ BigDecimal m = args.get(0);+ for (int i = 1; i < args.size(); i++) if (args.get(i).compareTo(m) > 0) m = args.get(i);+ return m;+ }++ private BigDecimal sum(List<BigDecimal> args) {+ if (args == null || args.isEmpty()) throw new IllegalArgumentException("sum requires at least one argument");+ BigDecimal s = BigDecimal.ZERO;+ for (BigDecimal v : args) s = s.add(v, MathContext.DECIMAL64);+ return s;+ }++ private BigDecimal mean(List<BigDecimal> args) {+ return sum(args).divide(new BigDecimal(args.size()), MathContext.DECIMAL64);+ }++ private char currentChar() {+ if (pos < 0 || pos >= expression.length()) return '\0';+ return expression.charAt(pos);+ }++ private boolean isIdentifierStart(char c) {+ return Character.isLetter(c) || c == '_';+ }++ private void skipWs() {+ while (pos < expression.length() && Character.isWhitespace(expression.charAt(pos))) pos++;+ }++ private boolean eat(char c) {+ if (pos < expression.length() && expression.charAt(pos) == c) {+ pos++;+ return true;+ }+ return false;+ }+ }+}diff --git a/source/net/yacy/ai/tools/ChitChatTool.java b/source/net/yacy/ai/tools/ChitChatTool.javanew file mode 100644index 000000000..8c0326068--- /dev/null+++ b/source/net/yacy/ai/tools/ChitChatTool.java@@ -0,0 +1,134 @@+/**+ * ChitChatTool+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.util.ArrayList;+import java.util.Collections;+import java.util.List;+import java.util.concurrent.ThreadLocalRandom;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.ai.ToolHandler;++public class ChitChatTool implements ToolHandler {++ private static final String NAME = "chitchat";+ private static final int DEFAULT_FACT_COUNT = 3;+ private static final int MAX_FACT_COUNT = 5;++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "Provide friendly chitchat. Call me when the user just tests the chat, says hello, is unsure with the prompt or in case of profanity.");++ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ JSONObject props = new JSONObject(true);++ JSONObject tone = new JSONObject(true);+ tone.put("type", "string");+ tone.put("description", "Optional tone hint, e.g. friendly, concise, playful.");+ props.put("tone", tone);++ JSONObject factCount = new JSONObject(true);+ factCount.put("type", "integer");+ factCount.put("description", "How many ability facts to include (1-5, default 3).");+ props.put("fact_count", factCount);++ params.put("properties", props);+ params.put("required", new JSONArray());+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }+++ public int maxCallsPerTurn() {+ return 1;+ }+++ public String execute(String arguments) {+ String tone = "friendly";+ int factCount = DEFAULT_FACT_COUNT;+ if (arguments != null && !arguments.isEmpty()) {+ try {+ JSONObject args = new JSONObject(arguments);+ tone = args.optString("tone", tone).trim();+ factCount = args.optInt("fact_count", factCount);+ } catch (JSONException e) {+ return ToolHandler.errorJson("Invalid arguments JSON");+ }+ }++ int count = clamp(factCount, 1, MAX_FACT_COUNT);+ List<String> selected = pickFacts(count);+ String message = buildMessage(tone, selected);++ try {+ JSONObject result = new JSONObject(true);+ result.put("tool", NAME);+ result.put("tone", tone);+ result.put("message", message);+ result.put("facts", new JSONArray(selected));+ return result.toString();+ } catch (JSONException e) {+ return ToolHandler.errorJson("Failed to build chitchat response");+ }+ }++ private static List<String> pickFacts(int count) {+ List<String> facts = new ArrayList<>(SelfReflectTool.facts());+ if (facts.isEmpty()) return facts;+ Collections.shuffle(facts, ThreadLocalRandom.current());+ int n = Math.min(count, facts.size());+ return new ArrayList<>(facts.subList(0, n));+ }++ private static String buildMessage(String tone, List<String> facts) {+ StringBuilder sb = new StringBuilder();+ sb.append("Happy to chat. ");+ sb.append(SelfReflectTool.purpose());+ if (tone != null && !tone.isEmpty()) {+ sb.append(" Tone requested: ").append(tone).append(". ");+ } else {+ sb.append(" ");+ }+ sb.append("Here are a few things I can do: ");+ for (int i = 0; i < facts.size(); i++) {+ if (i > 0) sb.append(" ");+ sb.append(i + 1).append(") ").append(facts.get(i));+ }+ return sb.toString();+ }++ private static int clamp(int value, int min, int max) {+ if (value < min) return min;+ if (value > max) return max;+ return value;+ }+}diff --git a/source/net/yacy/ai/tools/DateMathTool.java b/source/net/yacy/ai/tools/DateMathTool.javanew file mode 100644index 000000000..bcd3e333b--- /dev/null+++ b/source/net/yacy/ai/tools/DateMathTool.java@@ -0,0 +1,214 @@+/**+ * DateMathTool+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.time.DayOfWeek;+import java.time.Duration;+import java.time.LocalDate;+import java.time.LocalDateTime;+import java.time.YearMonth;+import java.time.ZoneId;+import java.time.ZonedDateTime;+import java.time.format.DateTimeFormatter;+import java.time.format.DateTimeParseException;+import java.time.temporal.ChronoUnit;+import java.util.Locale;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.ai.ToolHandler;++public class DateMathTool implements ToolHandler {++ private static final String NAME = "date_math";++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "Perform deterministic date/time operations: add, subtract, diff, start_of, end_of.");++ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ JSONObject props = new JSONObject(true);+ props.put("operation", new JSONObject(true).put("type", "string").put("description", "add|subtract|diff|start_of|end_of"));+ props.put("datetime", new JSONObject(true).put("type", "string").put("description", "Base datetime/date in ISO format."));+ props.put("other_datetime", new JSONObject(true).put("type", "string").put("description", "Second datetime for diff."));+ props.put("amount", new JSONObject(true).put("type", "integer").put("description", "Amount for add/subtract."));+ props.put("unit", new JSONObject(true).put("type", "string").put("description", "minute|hour|day|week|month|year"));+ props.put("period", new JSONObject(true).put("type", "string").put("description", "day|week|month|year (for start_of/end_of)."));+ props.put("timezone", new JSONObject(true).put("type", "string").put("description", "IANA zone; default system zone."));+ params.put("properties", props);+ params.put("required", new JSONArray().put("operation"));+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }+++ public int maxCallsPerTurn() {+ return 1;+ }+++ public String execute(String arguments) {+ final JSONObject args;+ try {+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);+ } catch (JSONException e) {+ return ToolHandler.errorJson("Invalid arguments JSON");+ }++ String op = args.optString("operation", "").trim().toLowerCase(Locale.ROOT);+ if (op.isEmpty()) return ToolHandler.errorJson("Missing operation");++ ZoneId zone = parseZone(args.optString("timezone", ""));+ if (zone == null) return ToolHandler.errorJson("Invalid timezone");+ ZonedDateTime base = parseDateTime(args.optString("datetime", ""), zone);+ if (base == null) base = ZonedDateTime.now(zone);++ try {+ JSONObject out = new JSONObject(true);+ out.put("tool", NAME);+ out.put("operation", op);+ out.put("timezone", zone.getId());+ out.put("input_datetime", base.toString());++ if ("add".equals(op) || "subtract".equals(op)) {+ int amount = args.optInt("amount", Integer.MIN_VALUE);+ if (amount == Integer.MIN_VALUE) return ToolHandler.errorJson("Missing amount");+ String unit = normalizeUnit(args.optString("unit", ""));+ if (unit == null) return ToolHandler.errorJson("Invalid unit");+ long signed = "subtract".equals(op) ? -((long) amount) : amount;+ ZonedDateTime result = applyAdd(base, signed, unit);+ out.put("result_datetime", result.toString());+ out.put("result_epoch_ms", result.toInstant().toEpochMilli());+ return out.toString();+ }++ if ("diff".equals(op)) {+ ZonedDateTime other = parseDateTime(args.optString("other_datetime", ""), zone);+ if (other == null) return ToolHandler.errorJson("Missing or invalid other_datetime");+ Duration d = Duration.between(base.toInstant(), other.toInstant());+ out.put("other_datetime", other.toString());+ out.put("difference_seconds", d.getSeconds());+ out.put("difference_minutes", d.toMinutes());+ out.put("difference_hours", d.toHours());+ out.put("difference_days", d.toDays());+ return out.toString();+ }++ if ("start_of".equals(op) || "end_of".equals(op)) {+ String period = args.optString("period", "").trim().toLowerCase(Locale.ROOT);+ if (period.isEmpty()) return ToolHandler.errorJson("Missing period");+ ZonedDateTime result = "start_of".equals(op) ? startOf(base, period) : endOf(base, period);+ out.put("period", period);+ out.put("result_datetime", result.toString());+ out.put("result_epoch_ms", result.toInstant().toEpochMilli());+ return out.toString();+ }+ return ToolHandler.errorJson("Unsupported operation: " + op);+ } catch (IllegalArgumentException | JSONException e) {+ return ToolHandler.errorJson(e.getMessage());+ }+ }++ private static ZoneId parseZone(String timezone) {+ if (timezone == null || timezone.trim().isEmpty()) return ZoneId.systemDefault();+ try {+ return ZoneId.of(timezone.trim());+ } catch (Exception e) {+ return null;+ }+ }++ private static ZonedDateTime parseDateTime(String value, ZoneId zone) {+ if (value == null || value.trim().isEmpty()) return null;+ String v = value.trim();+ try {+ return ZonedDateTime.parse(v);+ } catch (DateTimeParseException e) {}+ try {+ LocalDateTime ldt = LocalDateTime.parse(v, DateTimeFormatter.ISO_LOCAL_DATE_TIME);+ return ldt.atZone(zone);+ } catch (DateTimeParseException e) {}+ try {+ LocalDate ld = LocalDate.parse(v, DateTimeFormatter.ISO_LOCAL_DATE);+ return ld.atStartOfDay(zone);+ } catch (DateTimeParseException e) {}+ return null;+ }++ private static String normalizeUnit(String unit) {+ if (unit == null) return null;+ String u = unit.trim().toLowerCase(Locale.ROOT);+ if ("minutes".equals(u)) return "minute";+ if ("hours".equals(u)) return "hour";+ if ("days".equals(u)) return "day";+ if ("weeks".equals(u)) return "week";+ if ("months".equals(u)) return "month";+ if ("years".equals(u)) return "year";+ if ("minute".equals(u) || "hour".equals(u) || "day".equals(u) || "week".equals(u) || "month".equals(u) || "year".equals(u)) return u;+ return null;+ }++ private static ZonedDateTime applyAdd(ZonedDateTime dt, long amount, String unit) {+ if ("minute".equals(unit)) return dt.plus(amount, ChronoUnit.MINUTES);+ if ("hour".equals(unit)) return dt.plus(amount, ChronoUnit.HOURS);+ if ("day".equals(unit)) return dt.plus(amount, ChronoUnit.DAYS);+ if ("week".equals(unit)) return dt.plus(amount, ChronoUnit.WEEKS);+ if ("month".equals(unit)) return dt.plus(amount, ChronoUnit.MONTHS);+ if ("year".equals(unit)) return dt.plus(amount, ChronoUnit.YEARS);+ throw new IllegalArgumentException("Invalid unit: " + unit);+ }++ private static ZonedDateTime startOf(ZonedDateTime dt, String period) {+ if ("day".equals(period)) {+ return dt.toLocalDate().atStartOfDay(dt.getZone());+ }+ if ("week".equals(period)) {+ LocalDate date = dt.toLocalDate();+ while (date.getDayOfWeek() != DayOfWeek.MONDAY) date = date.minusDays(1);+ return date.atStartOfDay(dt.getZone());+ }+ if ("month".equals(period)) {+ LocalDate first = YearMonth.from(dt).atDay(1);+ return first.atStartOfDay(dt.getZone());+ }+ if ("year".equals(period)) {+ LocalDate first = LocalDate.of(dt.getYear(), 1, 1);+ return first.atStartOfDay(dt.getZone());+ }+ throw new IllegalArgumentException("Invalid period: " + period);+ }++ private static ZonedDateTime endOf(ZonedDateTime dt, String period) {+ if ("day".equals(period)) return startOf(dt, "day").plusDays(1).minusNanos(1);+ if ("week".equals(period)) return startOf(dt, "week").plusWeeks(1).minusNanos(1);+ if ("month".equals(period)) return startOf(dt, "month").plusMonths(1).minusNanos(1);+ if ("year".equals(period)) return startOf(dt, "year").plusYears(1).minusNanos(1);+ throw new IllegalArgumentException("Invalid period: " + period);+ }+}diff --git a/source/net/yacy/ai/tools/DateTimeTool.java b/source/net/yacy/ai/tools/DateTimeTool.javanew file mode 100644index 000000000..d9ba10d59--- /dev/null+++ b/source/net/yacy/ai/tools/DateTimeTool.java@@ -0,0 +1,112 @@+/**+ * DateTimeTool+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.time.Instant;+import java.time.ZoneId;+import java.time.ZonedDateTime;+import java.time.format.DateTimeFormatter;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.ai.ToolHandler;++public class DateTimeTool implements ToolHandler {++ private static final String NAME = "datetime";+ private static final DateTimeFormatter DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "Get the current date and time. Optionally specify an IANA timezone.");+ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ JSONObject props = new JSONObject(true);+ JSONObject tz = new JSONObject(true);+ tz.put("type", "string");+ tz.put("description", "IANA timezone name like America/New_York. Defaults to local time zone.");+ props.put("timezone", tz);+ params.put("properties", props);+ params.put("required", new JSONArray());+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }+++ public int maxCallsPerTurn() {+ return 1;+ }+++ public String execute(String arguments) {+ String timezone = null;+ if (arguments != null && !arguments.isEmpty()) {+ try {+ JSONObject obj = new JSONObject(arguments);+ timezone = obj.optString("timezone", null);+ } catch (JSONException e) {+ return ToolHandler.errorJson("Invalid arguments JSON");+ }+ }++ ZoneId localZone = ZoneId.systemDefault();+ ZoneId zone = localZone;+ String note = null;+ if (timezone != null && !timezone.isEmpty()) {+ try {+ zone = ZoneId.of(timezone);+ } catch (Exception e) {+ zone = localZone;+ note = "Invalid timezone \"" + timezone + "\". Used local timezone \"" + localZone.getId() + "\".";+ }+ }++ ZonedDateTime now = ZonedDateTime.now(zone);+ try {+ JSONObject result = new JSONObject(true);+ result.put("datetime", now.format(DATETIME_FORMAT));+ result.put("timezone", zone.getId());+ result.put("time_of_day", classifyTimeOfDay(now.getHour()));+ result.put("utc_offset_minutes", now.getOffset().getTotalSeconds() / 60);+ result.put("utc_datetime", Instant.now().toString());+ result.put("epoch_ms", System.currentTimeMillis());+ if (note != null) result.put("note", note);+ return result.toString();+ } catch (JSONException e) {+ return ToolHandler.errorJson("Failed to build datetime response");+ }+ }++ private static String classifyTimeOfDay(int hour) {+ if (hour < 0 || hour > 23) return "night";+ if (hour < 6) return "night";+ if (hour < 12) return "morning";+ if (hour < 18) return "afternoon";+ return "evening";+ }+}diff --git a/source/net/yacy/ai/tools/HttpJsonTool.java b/source/net/yacy/ai/tools/HttpJsonTool.javanew file mode 100644index 000000000..16251b3e5--- /dev/null+++ b/source/net/yacy/ai/tools/HttpJsonTool.java@@ -0,0 +1,244 @@+/**+ * HttpJsonTool+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.HttpURLConnection;+import java.net.URI;+import java.net.URISyntaxException;+import java.net.URL;+import java.nio.charset.StandardCharsets;+import java.util.Arrays;+import java.util.HashSet;+import java.util.Set;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;+import org.json.JSONTokener;++import net.yacy.ai.ToolHandler;+import net.yacy.cora.document.id.DigestURL;++public class HttpJsonTool implements ToolHandler {++ private static final String NAME = "http_json";+ private static final Set<String> ALLOWED_METHODS = new HashSet<>(Arrays.asList("GET", "POST", "PUT", "PATCH", "DELETE"));+ private static final Set<String> BLOCKED_HEADERS = new HashSet<>(Arrays.asList("host", "content-length", "connection", "transfer-encoding"));+ private static final int DEFAULT_TIMEOUT_MS = 15000;+ private static final int MAX_TIMEOUT_MS = 60000;+ private static final int DEFAULT_MAX_BYTES = 1_000_000;+ private static final int MAX_ALLOWED_BYTES = 2_000_000;++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "Fetch an HTTP endpoint and return parsed JSON response.");++ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ JSONObject props = new JSONObject(true);++ JSONObject url = new JSONObject(true);+ url.put("type", "string");+ url.put("description", "Absolute http or https URL.");+ props.put("url", url);++ JSONObject method = new JSONObject(true);+ method.put("type", "string");+ method.put("description", "HTTP method. Allowed: GET, POST, PUT, PATCH, DELETE.");+ props.put("method", method);++ JSONObject headers = new JSONObject(true);+ headers.put("type", "object");+ headers.put("description", "Optional request headers (string values).");+ props.put("headers", headers);++ JSONObject body = new JSONObject(true);+ body.put("description", "Optional JSON body as object/array/string.");+ props.put("body", body);++ JSONObject timeout = new JSONObject(true);+ timeout.put("type", "integer");+ timeout.put("description", "Timeout in milliseconds (max 60000).");+ props.put("timeout_ms", timeout);++ JSONObject maxBytes = new JSONObject(true);+ maxBytes.put("type", "integer");+ maxBytes.put("description", "Maximum response size in bytes (max 2000000).");+ props.put("max_bytes", maxBytes);++ params.put("properties", props);+ params.put("required", new JSONArray().put("url"));+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }+++ public int maxCallsPerTurn() {+ return 1;+ }+++ public String execute(String arguments) {+ final JSONObject args;+ try {+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);+ } catch (JSONException e) {+ return ToolHandler.errorJson("Invalid arguments JSON. don't try again");+ }++ final String urlRaw = args.optString("url", "").trim();+ if (urlRaw.isEmpty()) return ToolHandler.errorJson("Missing url. don't try again");++ final DigestURL digestUrl;+ try {+ digestUrl = new DigestURL(urlRaw);+ } catch (Exception e) {+ return ToolHandler.errorJson("Invalid URL. don't try again");+ }+ final String protocol = digestUrl.getProtocol();+ if (!"http".equalsIgnoreCase(protocol) && !"https".equalsIgnoreCase(protocol)) {+ return ToolHandler.errorJson("Only http/https URLs are allowed. don't try again");+ }++ final String method = args.optString("method", "GET").trim().toUpperCase();+ if (!ALLOWED_METHODS.contains(method)) {+ return ToolHandler.errorJson("Unsupported method " + method + ". don't try again");+ }+ final int timeoutMs = clampPositive(args.optInt("timeout_ms", DEFAULT_TIMEOUT_MS), DEFAULT_TIMEOUT_MS, MAX_TIMEOUT_MS);+ final int maxBytes = clampPositive(args.optInt("max_bytes", DEFAULT_MAX_BYTES), DEFAULT_MAX_BYTES, MAX_ALLOWED_BYTES);+ final Object body = args.opt("body");+ final JSONObject headers = args.optJSONObject("headers");++ HttpURLConnection conn = null;+ try {+ final URL url = new URI(digestUrl.toNormalform(true)).toURL();+ conn = (HttpURLConnection) url.openConnection();+ conn.setInstanceFollowRedirects(true);+ conn.setRequestMethod(method);+ conn.setConnectTimeout(timeoutMs);+ conn.setReadTimeout(timeoutMs);+ conn.setRequestProperty("Accept", "application/json");+ applyHeaders(conn, headers);++ if (body != null && !"GET".equals(method)) {+ conn.setDoOutput(true);+ if (conn.getRequestProperty("Content-Type") == null) {+ conn.setRequestProperty("Content-Type", "application/json");+ }+ byte[] payload = encodeBody(body);+ try (OutputStream os = conn.getOutputStream()) {+ os.write(payload);+ }+ }++ int status = conn.getResponseCode();+ String contentType = conn.getHeaderField("Content-Type");+ if (contentType == null) contentType = "";+ final InputStream stream = status >= 400 ? conn.getErrorStream() : conn.getInputStream();+ if (stream == null) {+ return ToolHandler.errorJson("Empty response body. don't try again");+ }+ final String text;+ try (InputStream in = stream) {+ text = new String(readLimited(in, maxBytes), StandardCharsets.UTF_8);+ }+ if (text.trim().isEmpty()) {+ return ToolHandler.errorJson("Empty response body. don't try again");+ }++ final Object parsed;+ try {+ parsed = new JSONTokener(text).nextValue();+ } catch (JSONException e) {+ return ToolHandler.errorJson("Response is not valid JSON. don't try again");+ }+ if (!(parsed instanceof JSONObject) && !(parsed instanceof JSONArray)) {+ return ToolHandler.errorJson("Response is not JSON object/array. don't try again");+ }++ JSONObject result = new JSONObject(true);+ result.put("url", digestUrl.toNormalform(true));+ result.put("method", method);+ result.put("status", status);+ result.put("content_type", contentType);+ result.put("json", parsed);+ return result.toString();+ } catch (IOException e) {+ return ToolHandler.errorJson("Fetch error: " + e.getMessage() + ". don't try again");+ } catch (URISyntaxException e) {+ return ToolHandler.errorJson("Invalid URL syntax. don't try again");+ } catch (JSONException e) {+ return ToolHandler.errorJson("Failed to build tool response. don't try again");+ } finally {+ if (conn != null) conn.disconnect();+ }+ }++ private static void applyHeaders(HttpURLConnection conn, JSONObject headers) {+ if (conn == null || headers == null) return;+ for (String key : headers.keySet()) {+ if (key == null) continue;+ final String normalized = key.trim();+ if (normalized.isEmpty()) continue;+ if (BLOCKED_HEADERS.contains(normalized.toLowerCase())) continue;+ final String value = headers.optString(key, "");+ conn.setRequestProperty(normalized, value);+ }+ }++ private static byte[] encodeBody(Object body) {+ if (body == null) return new byte[0];+ if (body instanceof JSONObject) return ((JSONObject) body).toString().getBytes(StandardCharsets.UTF_8);+ if (body instanceof JSONArray) return ((JSONArray) body).toString().getBytes(StandardCharsets.UTF_8);+ if (body instanceof String) return ((String) body).getBytes(StandardCharsets.UTF_8);+ return String.valueOf(body).getBytes(StandardCharsets.UTF_8);+ }++ private static byte[] readLimited(InputStream in, int maxBytes) throws IOException {+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ byte[] buffer = new byte[4096];+ int total = 0;+ int read;+ while ((read = in.read(buffer)) != -1) {+ total += read;+ if (total > maxBytes) {+ throw new IOException("Response too large");+ }+ out.write(buffer, 0, read);+ }+ return out.toByteArray();+ }++ private static int clampPositive(int value, int defaultValue, int maxValue) {+ if (value <= 0) return defaultValue;+ if (value > maxValue) return maxValue;+ return value;+ }+}diff --git a/source/net/yacy/ai/tools/NumberParserTool.java b/source/net/yacy/ai/tools/NumberParserTool.javanew file mode 100644index 000000000..9880de5e5--- /dev/null+++ b/source/net/yacy/ai/tools/NumberParserTool.java@@ -0,0 +1,403 @@+/**+ * NumberParserTool+ * Copyright 2026 by Michael Peter Christen+ * First released 07.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.math.BigDecimal;+import java.text.Normalizer;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.HashMap;+import java.util.List;+import java.util.Locale;+import java.util.Map;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.ai.ToolHandler;++public class NumberParserTool implements ToolHandler {++ private static final String NAME = "number_parser";++ private enum Lang {+ DE, EN, FR, ES, IT+ }++ private static final class Lexicon {+ final Map<String, Integer> small = new HashMap<>();+ final Map<String, Integer> scales = new HashMap<>();+ final String decimalMarker;+ Lexicon(String decimalMarker) {+ this.decimalMarker = decimalMarker;+ }+ }++ private static final Map<Lang, Lexicon> LEX = buildLexicons();++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "Parse written number words into numeric values. Supports de/en/fr/es/it.");++ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ JSONObject props = new JSONObject(true);++ props.put("text", new JSONObject(true)+ .put("type", "string")+ .put("description", "Number words, e.g. einundzwanzig, five hundred seven, vingt-et-un."));++ props.put("language", new JSONObject(true)+ .put("type", "string")+ .put("description", "Optional: de|en|fr|es|it|auto (default auto)."));++ props.put("strict", new JSONObject(true)+ .put("type", "boolean")+ .put("description", "If true (default), all tokens must be recognized as number words."));++ params.put("properties", props);+ params.put("required", new JSONArray().put("text"));+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }++ @Override+ public int maxCallsPerTurn() {+ return 10;+ }++ @Override+ public String execute(String arguments) {+ final JSONObject args;+ try {+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);+ } catch (JSONException e) {+ return ToolHandler.errorJson("Invalid arguments JSON");+ }++ final String text = args.optString("text", "").trim();+ if (text.isEmpty()) return ToolHandler.errorJson("Missing text");++ final String language = args.optString("language", "auto").trim().toLowerCase(Locale.ROOT);+ final boolean strict = args.optBoolean("strict", true);++ ParseResult best = null;+ Lang bestLang = null;++ if (!"auto".equals(language)) {+ Lang lang = parseLang(language);+ if (lang == null) return ToolHandler.errorJson("Invalid language. Use de|en|fr|es|it|auto");+ ParseResult r = parseWithLang(text, lang, strict);+ if (r == null) return ToolHandler.errorJson("Could not parse number words");+ best = r;+ bestLang = lang;+ } else {+ for (Lang lang : Lang.values()) {+ ParseResult r = parseWithLang(text, lang, strict);+ if (r == null) continue;+ if (best == null || r.score > best.score) {+ best = r;+ bestLang = lang;+ }+ }+ if (best == null) return ToolHandler.errorJson("Could not parse number words in supported languages");+ }++ try {+ JSONObject out = new JSONObject(true);+ out.put("tool", NAME);+ out.put("text", text);+ out.put("language", bestLang.name().toLowerCase(Locale.ROOT));+ out.put("value", best.value.stripTrailingZeros().toPlainString());+ return out.toString();+ } catch (JSONException e) {+ return ToolHandler.errorJson("Failed to build number parser response");+ }+ }++ private static Lang parseLang(String code) {+ if ("de".equals(code)) return Lang.DE;+ if ("en".equals(code)) return Lang.EN;+ if ("fr".equals(code)) return Lang.FR;+ if ("es".equals(code)) return Lang.ES;+ if ("it".equals(code)) return Lang.IT;+ return null;+ }++ private static final class ParseResult {+ final BigDecimal value;+ final int score;+ ParseResult(BigDecimal value, int score) {+ this.value = value;+ this.score = score;+ }+ }++ private static ParseResult parseWithLang(String input, Lang lang, boolean strict) {+ String normalized = normalize(input);+ List<String> tokens = tokenizeByLanguage(normalized, lang);+ if (tokens.isEmpty()) return null;++ Lexicon lx = LEX.get(lang);+ if (lx == null) return null;++ int decimalIndex = tokens.indexOf(lx.decimalMarker);+ BigDecimal intPart;+ int consumedInt;+ if (decimalIndex >= 0) {+ ParseSpan left = parseIntegerTokens(tokens.subList(0, decimalIndex), lang, strict);+ if (left == null) return null;+ intPart = new BigDecimal(left.value);+ consumedInt = left.consumed;++ ParseFraction frac = parseFractionTokens(tokens.subList(decimalIndex + 1, tokens.size()), lang, strict);+ if (frac == null) return null;+ BigDecimal value = intPart.add(frac.value);+ int score = consumedInt + frac.consumed + 1;+ return new ParseResult(value, score);+ }++ ParseSpan span = parseIntegerTokens(tokens, lang, strict);+ if (span == null) return null;+ return new ParseResult(new BigDecimal(span.value), span.consumed);+ }++ private static final class ParseSpan {+ final long value;+ final int consumed;+ ParseSpan(long value, int consumed) {+ this.value = value;+ this.consumed = consumed;+ }+ }++ private static ParseSpan parseIntegerTokens(List<String> tokens, Lang lang, boolean strict) {+ if (tokens.isEmpty()) return new ParseSpan(0L, 0);+ Lexicon lx = LEX.get(lang);+ long total = 0L;+ long current = 0L;+ int consumed = 0;++ for (int i = 0; i < tokens.size(); i++) {+ String t = tokens.get(i);+ if (t == null || t.isEmpty() || "and".equals(t) || "et".equals(t) || "y".equals(t) || "und".equals(t) || "e".equals(t)) {+ continue;+ }++ if (isDigits(t)) {+ long v = Long.parseLong(t);+ current += v;+ consumed++;+ continue;+ }++ // French special: quatre vingt => 80+ if (lang == Lang.FR && "quatre".equals(t) && i + 1 < tokens.size() && "vingt".equals(tokens.get(i + 1))) {+ current += 80;+ consumed += 2;+ i++;+ continue;+ }++ Integer small = lx.small.get(t);+ if (small != null) {+ current += small.intValue();+ consumed++;+ continue;+ }++ if ("hundred".equals(t) || "hundert".equals(t) || "cent".equals(t) || "cien".equals(t) || "ciento".equals(t) || "cento".equals(t)) {+ if (current == 0) current = 1;+ current *= 100;+ consumed++;+ continue;+ }++ Integer scale = lx.scales.get(t);+ if (scale != null) {+ if (current == 0) current = 1;+ total += current * scale.longValue();+ current = 0;+ consumed++;+ continue;+ }++ if (strict) return null;+ }+ return new ParseSpan(total + current, consumed);+ }++ private static final class ParseFraction {+ final BigDecimal value;+ final int consumed;+ ParseFraction(BigDecimal value, int consumed) {+ this.value = value;+ this.consumed = consumed;+ }+ }++ private static ParseFraction parseFractionTokens(List<String> tokens, Lang lang, boolean strict) {+ if (tokens.isEmpty()) return new ParseFraction(BigDecimal.ZERO, 0);+ Lexicon lx = LEX.get(lang);+ StringBuilder digits = new StringBuilder();+ int consumed = 0;+ for (String t : tokens) {+ if (t == null || t.isEmpty()) continue;+ if (isDigits(t)) {+ digits.append(t);+ consumed++;+ continue;+ }+ Integer v = lx.small.get(t);+ if (v != null && v.intValue() >= 0 && v.intValue() <= 9) {+ digits.append(v.intValue());+ consumed++;+ continue;+ }+ if (strict) return null;+ }+ if (digits.length() == 0) return new ParseFraction(BigDecimal.ZERO, consumed);+ BigDecimal frac = new BigDecimal("0." + digits.toString());+ return new ParseFraction(frac, consumed);+ }++ private static boolean isDigits(String t) {+ if (t == null || t.isEmpty()) return false;+ for (int i = 0; i < t.length(); i++) if (!Character.isDigit(t.charAt(i))) return false;+ return true;+ }++ private static String normalize(String s) {+ String n = s == null ? "" : s.toLowerCase(Locale.ROOT);+ n = n.replace('’', '\'');+ n = n.replace('-', ' ');+ n = n.replace("ß", "ss");+ n = Normalizer.normalize(n, Normalizer.Form.NFD).replaceAll("\\p{M}+", "");+ n = n.replaceAll("[^a-z0-9' ]", " ");+ n = n.replaceAll("\\s+", " ").trim();+ return n;+ }++ private static List<String> tokenizeByLanguage(String normalized, Lang lang) {+ if (normalized.isEmpty()) return new ArrayList<>();+ String s = normalized;++ if (lang == Lang.DE) {+ // Split common German compounding boundaries in one pass to avoid+ // destructive re-replacements (e.g. "hundert" -> "... t").+ s = s.replaceAll("(millionen|million)", " million ");+ s = s.replaceAll("(milliarden|milliarde)", " milliarde ");+ s = s.replaceAll("(tausend)", " tausend ");+ s = s.replaceAll("(hundert|hunder)", " hundert ");+ s = s.replace("tausend", " tausend ");+ // Split forms like "hundertundsieben" -> "hundert und sieben".+ s = s.replaceAll("\\bund(ein|eine|eins|zwei|drei|vier|funf|fuenf|sechs|sieben|acht|neun|zehn|elf|zwolf|zwoelf|dreizehn|vierzehn|funfzehn|fuenfzehn|sechzehn|siebzehn|achtzehn|neunzehn|zwanzig|dreissig|dreiig|vierzig|funfzig|fuenfzig|sechzig|siebzig|achtzig|neunzig)\\b", "und $1");+ // Split only canonical DE one-and-tens compounds (e.g. einundzwanzig).+ s = s.replaceAll("(ein|eine|zwei|drei|vier|funf|fuenf|sechs|sieben|acht|neun)und(zwanzig|dreissig|dreiig|vierzig|funfzig|fuenfzig|sechzig|siebzig|achtzig|neunzig)", "$1 und $2");+ s = s.replace("eins", "ein");+ }++ if (lang == Lang.ES) {+ s = s.replaceAll("veinti([a-z]+)", "veinte $1");+ s = s.replaceAll("dieci([a-z]+)", "diez $1");+ }++ if (lang == Lang.IT) {+ s = s.replaceAll("(venti|trenta|quaranta|cinquanta|sessanta|settanta|ottanta|novanta)([a-z]+)", "$1 $2");+ }++ if (lang == Lang.FR) {+ s = s.replace("quatre vingt", "quatre vingt");+ }++ s = s.replaceAll("\\s+", " ").trim();+ if (s.isEmpty()) return new ArrayList<>();+ return new ArrayList<>(Arrays.asList(s.split(" ")));+ }++ private static Map<Lang, Lexicon> buildLexicons() {+ Map<Lang, Lexicon> map = new HashMap<>();++ Lexicon de = new Lexicon("komma");+ putAll(de.small,+ "null",0, "ein",1, "eine",1, "einen",1, "eins",1, "zwo",2, "zwei",2, "drei",3, "vier",4, "funf",5, "fuenf",5, "sechs",6,+ "sieben",7, "acht",8, "neun",9, "zehn",10, "elf",11, "zwolf",12, "zwoelf",12, "dreizehn",13, "vierzehn",14,+ "funfzehn",15, "fuenfzehn",15, "sechzehn",16, "siebzehn",17, "achtzehn",18, "neunzehn",19,+ "zwanzig",20, "dreissig",30, "dreiig",30, "vierzig",40, "funfzig",50, "fuenfzig",50, "sechzig",60, "siebzig",70,+ "achtzig",80, "neunzig",90);+ putAll(de.scales, "tausend",1000, "million",1000000, "millionen",1000000, "milliarde",1000000000, "milliarden",1000000000);+ map.put(Lang.DE, de);++ Lexicon en = new Lexicon("point");+ putAll(en.small,+ "zero",0, "one",1, "two",2, "three",3, "four",4, "five",5, "six",6, "seven",7, "eight",8, "nine",9,+ "ten",10, "eleven",11, "twelve",12, "thirteen",13, "fourteen",14, "fifteen",15, "sixteen",16,+ "seventeen",17, "eighteen",18, "nineteen",19,+ "twenty",20, "thirty",30, "forty",40, "fifty",50, "sixty",60, "seventy",70, "eighty",80, "ninety",90);+ putAll(en.scales, "thousand",1000, "million",1000000, "billion",1000000000);+ map.put(Lang.EN, en);++ Lexicon fr = new Lexicon("virgule");+ putAll(fr.small,+ "zero",0, "un",1, "une",1, "deux",2, "trois",3, "quatre",4, "cinq",5, "six",6, "sept",7, "huit",8, "neuf",9,+ "dix",10, "onze",11, "douze",12, "treize",13, "quatorze",14, "quinze",15, "seize",16,+ "vingt",20, "trente",30, "quarante",40, "cinquante",50, "soixante",60, "soixantedix",70,+ "quatrevingt",80, "quatrevingtdix",90);+ putAll(fr.scales, "mille",1000, "million",1000000, "milliard",1000000000);+ map.put(Lang.FR, fr);++ Lexicon es = new Lexicon("coma");+ putAll(es.small,+ "cero",0, "uno",1, "una",1, "dos",2, "tres",3, "cuatro",4, "cinco",5, "seis",6, "siete",7, "ocho",8, "nueve",9,+ "diez",10, "once",11, "doce",12, "trece",13, "catorce",14, "quince",15, "dieciseis",16,+ "diecisiete",17, "dieciocho",18, "diecinueve",19,+ "veinte",20, "treinta",30, "cuarenta",40, "cincuenta",50, "sesenta",60, "setenta",70, "ochenta",80, "noventa",90,+ "cien",100, "ciento",100, "doscientos",200, "trescientos",300, "cuatrocientos",400, "quinientos",500,+ "seiscientos",600, "setecientos",700, "ochocientos",800, "novecientos",900);+ putAll(es.scales, "mil",1000, "millon",1000000, "millones",1000000, "milmillon",1000000000);+ map.put(Lang.ES, es);++ Lexicon it = new Lexicon("virgola");+ putAll(it.small,+ "zero",0, "uno",1, "una",1, "due",2, "tre",3, "quattro",4, "cinque",5, "sei",6, "sette",7, "otto",8, "nove",9,+ "dieci",10, "undici",11, "dodici",12, "tredici",13, "quattordici",14, "quindici",15, "sedici",16,+ "diciassette",17, "diciotto",18, "diciannove",19,+ "venti",20, "trenta",30, "quaranta",40, "cinquanta",50, "sessanta",60, "settanta",70, "ottanta",80, "novanta",90,+ "cento",100, "duecento",200, "trecento",300, "quattrocento",400, "cinquecento",500,+ "seicento",600, "settecento",700, "ottocento",800, "novecento",900);+ putAll(it.scales, "mille",1000, "mila",1000, "milione",1000000, "milioni",1000000, "miliardo",1000000000);+ map.put(Lang.IT, it);++ return map;+ }++ private static void putAll(Map<String, Integer> m, Object... kv) {+ for (int i = 0; i + 1 < kv.length; i += 2) {+ m.put(String.valueOf(kv[i]), Integer.valueOf(((Number) kv[i + 1]).intValue()));+ }+ }+}diff --git a/source/net/yacy/ai/tools/PromptToMermaidTool.java b/source/net/yacy/ai/tools/PromptToMermaidTool.javanew file mode 100644index 000000000..dc685f0d9--- /dev/null+++ b/source/net/yacy/ai/tools/PromptToMermaidTool.java@@ -0,0 +1,314 @@+/**+ * PromptToMermaidTool+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.io.IOException;+import java.util.Arrays;+import java.util.List;+import java.util.Locale;+import java.util.regex.Pattern;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;+import org.json.JSONTokener;++import net.yacy.ai.ToolHandler;+import net.yacy.ai.LLM;++public class PromptToMermaidTool implements ToolHandler {++ private static final String NAME = "prompt_to_mermaid";+ //private static final List<String> DIAGRAM_TYPES = Arrays.asList("flowchart", "sequence", "graph");++ private static final String SYSTEM_PROMPT =+ "You convert natural-language process descriptions into Mermaid diagrams with deterministic, transparent behavior. "+ + "Prefer explicit structure only from user text. Do not invent hidden steps. "+ + "Support flowchart, sequence diagram, and graph only. "+ + "If prompt is ambiguous or too vague, return fallback with plain-language summary and no forced diagram. "+ + "Return ONLY JSON matching the requested fields.";++ private static final JSONObject RESPONSE_SCHEMA = buildResponseSchema();++ private static final Pattern EMAIL_PATTERN = Pattern.compile("(?i)\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}\\b");+ private static final Pattern PHONE_PATTERN = Pattern.compile("\\b\\+?[0-9][0-9\\s().-]{6,}[0-9]\\b");++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "Convert a user prompt into a Mermaid diagram with fallback summary when prompt is ambiguous.");++ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ JSONObject props = new JSONObject(true);++ JSONObject prompt = new JSONObject(true);+ prompt.put("type", "string");+ prompt.put("description", "Natural-language diagram request.");+ props.put("prompt", prompt);++ JSONObject preferredType = new JSONObject(true);+ preferredType.put("type", "string");+ preferredType.put("description", "Optional hint: flowchart, sequence, graph.");+ props.put("preferred_diagram_type", preferredType);++ params.put("properties", props);+ params.put("required", new JSONArray().put("prompt"));+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }+++ public int maxCallsPerTurn() {+ return 1;+ }+++ public String execute(String arguments) {+ final JSONObject args;+ try {+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);+ } catch (JSONException e) {+ return fallback("Invalid arguments JSON", "", null, Arrays.asList("Could not parse tool arguments"));+ }++ String prompt = args.optString("prompt", "").trim();+ String preferredType = normalizeType(args.optString("preferred_diagram_type", null));+ if (prompt.isEmpty()) {+ return fallback("Missing prompt", "", preferredType, Arrays.asList("No prompt text provided"));+ }++ String redactedPrompt = redactSensitive(prompt);+ if (isVague(redactedPrompt)) {+ return fallback("Prompt is too vague for deterministic diagram generation", redactedPrompt, preferredType,+ Arrays.asList("Add concrete steps, actors, or relationships", "Fallback used to avoid hallucinated structure"));+ }++ LLM.LLMModel llmModel = LLM.llmFromUsage(LLM.LLMUsage.chat);+ if (llmModel == null || llmModel.llm == null || llmModel.model == null || llmModel.model.isEmpty()) {+ return fallback("No chat model configured for Mermaid generation", redactedPrompt, preferredType,+ Arrays.asList("Configure a chat model in AI production models"));+ }++ final String userPrompt = buildUserPrompt(redactedPrompt, preferredType);+ try {+ String raw = queryModel(llmModel, userPrompt, RESPONSE_SCHEMA);+ JSONObject parsed = parseResponseObject(raw);+ return normalizeResult(parsed, redactedPrompt, preferredType).toString();+ } catch (IOException e) {+ try {+ String raw = queryModel(llmModel, userPrompt, null);+ JSONObject parsed = parseResponseObject(raw);+ return normalizeResult(parsed, redactedPrompt, preferredType).toString();+ } catch (Exception retryError) {+ return fallback("LLM generation failed", redactedPrompt, preferredType,+ Arrays.asList("Model call failed: " + retryError.getMessage()));+ }+ } catch (Exception e) {+ return fallback("Invalid model output", redactedPrompt, preferredType,+ Arrays.asList("Failed to parse model output: " + e.getMessage()));+ }+ }++ private static String queryModel(LLM.LLMModel llmModel, String userPrompt, JSONObject schema) throws IOException {+ try {+ LLM.Context context = new LLM.Context(SYSTEM_PROMPT);+ context.addPrompt(userPrompt);+ return llmModel.llm.chat(llmModel.model, context, schema, 1200);+ } catch (JSONException e) {+ throw new IOException(e.getMessage(), e);+ }+ }++ private static JSONObject parseResponseObject(String raw) throws JSONException {+ if (raw == null) throw new JSONException("Empty model response");+ String trimmed = raw.trim();+ if (trimmed.startsWith("```")) {+ int firstNewline = trimmed.indexOf('\n');+ if (firstNewline >= 0) {+ trimmed = trimmed.substring(firstNewline + 1);+ }+ int lastFence = trimmed.lastIndexOf("```");+ if (lastFence >= 0) {+ trimmed = trimmed.substring(0, lastFence).trim();+ }+ }+ return new JSONObject(new JSONTokener(trimmed));+ }++ private static JSONObject normalizeResult(JSONObject parsed, String prompt, String preferredType) throws JSONException {+ String status = parsed.optString("status", "fallback").trim().toLowerCase(Locale.ROOT);+ if (!"success".equals(status) && !"fallback".equals(status)) status = "fallback";++ String diagramType = normalizeType(parsed.optString("diagram_type", null));+ String mermaidCode = parsed.optString("mermaid_code", "").trim();+ String reasoning = parsed.optString("reasoning", "").trim();+ JSONArray warnings = parsed.optJSONArray("warnings");+ if (warnings == null) warnings = new JSONArray();++ if (reasoning.isEmpty()) {+ reasoning = "Generated from prompt using local structured conversion rules and LLM formatting.";+ }++ if ("success".equals(status)) {+ if (diagramType == null) {+ status = "fallback";+ warnings.put("diagram_type missing or invalid");+ }+ if (!isValidMermaid(mermaidCode, diagramType)) {+ status = "fallback";+ warnings.put("Invalid Mermaid syntax for declared diagram_type");+ }+ }++ if ("fallback".equals(status)) {+ String fallbackReason = reasoning.isEmpty() ? "Prompt could not be deterministically mapped to a clear diagram." : reasoning;+ return buildResult("fallback", null, "", fallbackReason, warnings, prompt, preferredType);+ }+ return buildResult("success", diagramType, mermaidCode, reasoning, warnings, prompt, preferredType);+ }++ private static JSONObject buildResult(String status, String diagramType, String mermaidCode, String reasoning,+ JSONArray warnings, String prompt, String preferredType) throws JSONException {+ JSONObject out = new JSONObject(true);+ out.put("status", status);+ out.put("diagram_type", diagramType == null ? JSONObject.NULL : diagramType);+ out.put("mermaid_code", mermaidCode == null ? "" : mermaidCode);+ out.put("reasoning", reasoning == null ? "" : reasoning);+ out.put("warnings", warnings == null ? new JSONArray() : warnings);+ if (prompt != null && !prompt.isEmpty()) out.put("prompt_redacted", prompt);+ if (preferredType != null) out.put("preferred_diagram_type", preferredType);+ return out;+ }++ private static boolean isValidMermaid(String code, String diagramType) {+ if (code == null || code.trim().isEmpty() || diagramType == null) return false;+ String text = code.trim().toLowerCase(Locale.ROOT);+ if ("flowchart".equals(diagramType)) return text.startsWith("flowchart");+ if ("sequence".equals(diagramType)) return text.startsWith("sequencediagram");+ if ("graph".equals(diagramType)) return text.startsWith("graph");+ return false;+ }++ private static String buildUserPrompt(String prompt, String preferredType) {+ StringBuilder sb = new StringBuilder();+ sb.append("Convert the following user request into Mermaid.\n");+ sb.append("Requirements:\n");+ sb.append("- deterministic interpretation only\n");+ sb.append("- supported diagram types: flowchart, sequence, graph\n");+ sb.append("- if ambiguous, return fallback with summary reasoning\n");+ sb.append("- include warnings for assumptions\n");+ sb.append("- output JSON only with keys: status, diagram_type, mermaid_code, reasoning, warnings\n");+ if (preferredType != null) {+ sb.append("- user preferred diagram type: ").append(preferredType).append("\n");+ }+ sb.append("\nUser prompt:\n");+ sb.append(prompt);+ return sb.toString();+ }++ private static boolean isVague(String prompt) {+ if (prompt == null) return true;+ String trimmed = prompt.trim();+ if (trimmed.length() < 24) return true;+ String[] tokens = trimmed.split("\\s+");+ if (tokens.length < 6) return true;+ String lower = trimmed.toLowerCase(Locale.ROOT);+ return !(lower.contains("then") || lower.contains("first") || lower.contains("after")+ || lower.contains("if") || lower.contains("when") || lower.contains("->")+ || lower.contains("sequence") || lower.contains("workflow") || lower.contains("relationship"));+ }++ private static String redactSensitive(String prompt) {+ if (prompt == null) return "";+ String s = EMAIL_PATTERN.matcher(prompt).replaceAll("[EMAIL]");+ s = PHONE_PATTERN.matcher(s).replaceAll("[PHONE]");+ return s;+ }++ private static String normalizeType(String value) {+ if (value == null) return null;+ String type = value.trim().toLowerCase(Locale.ROOT);+ if (type.isEmpty()) return null;+ if ("flowchart".equals(type) || "sequence".equals(type) || "graph".equals(type)) return type;+ if ("sequence diagram".equals(type) || "sequencediagram".equals(type)) return "sequence";+ if ("flow".equals(type)) return "flowchart";+ return null;+ }++ private static String fallback(String reason, String prompt, String preferredType, List<String> warnings) {+ try {+ JSONArray w = new JSONArray();+ if (warnings != null) {+ for (String warning : warnings) {+ if (warning != null && !warning.isEmpty()) w.put(warning);+ }+ }+ return buildResult("fallback", null, "",+ reason == null ? "Fallback used due to insufficient structure in prompt." : reason,+ w, prompt, preferredType).toString();+ } catch (JSONException e) {+ return ToolHandler.errorJson("Failed to build fallback response");+ }+ }++ private static JSONObject buildResponseSchema() {+ try {+ JSONObject schema = new JSONObject(true);+ schema.put("title", "PromptToMermaidResult");+ schema.put("type", "object");+ JSONObject properties = new JSONObject(true);++ JSONObject status = new JSONObject(true);+ status.put("type", "string");+ status.put("enum", new JSONArray().put("success").put("fallback"));+ properties.put("status", status);++ JSONObject diagramType = new JSONObject(true);+ diagramType.put("type", "string");+ diagramType.put("enum", new JSONArray().put("flowchart").put("sequence").put("graph"));+ properties.put("diagram_type", diagramType);++ JSONObject mermaid = new JSONObject(true);+ mermaid.put("type", "string");+ properties.put("mermaid_code", mermaid);++ JSONObject reasoning = new JSONObject(true);+ reasoning.put("type", "string");+ properties.put("reasoning", reasoning);++ JSONObject warnings = new JSONObject(true);+ warnings.put("type", "array");+ warnings.put("items", new JSONObject(true).put("type", "string"));+ properties.put("warnings", warnings);++ schema.put("properties", properties);+ schema.put("required", new JSONArray().put("status").put("diagram_type").put("mermaid_code").put("reasoning").put("warnings"));+ return schema;+ } catch (JSONException e) {+ return null;+ }+ }+}diff --git a/source/net/yacy/ai/tools/SelfReflectTool.java b/source/net/yacy/ai/tools/SelfReflectTool.javanew file mode 100644index 000000000..d1f99d099--- /dev/null+++ b/source/net/yacy/ai/tools/SelfReflectTool.java@@ -0,0 +1,115 @@+/**+ * SelfReflectTool+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.util.Arrays;+import java.util.Collections;+import java.util.List;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.ai.ToolHandler;++public class SelfReflectTool implements ToolHandler {++ private static final String NAME = "self_reflect";++ private static final String PURPOSE =+ "YaCy AI aims to be the good one: privacy-aware, open, and useful AI that combines LLM responses with transparent tools and retrieval from YaCy search.";++ private static final List<String> FACTS = Collections.unmodifiableList(Arrays.asList(+ "You can choose default dialog augmentation mode: no search, local search, or global search.",+ "You can click the search button for the current message to attach search results as context without uploading a file.",+ "You can run with no search and attach your own files instead, so you control exactly what context is sent.",+ "You can attach PNG/JPG images and text files (.txt, .md, .tex), preview them, download them, or open them in a full view.",+ "You can drag and drop supported files directly into the composer.",+ "You can see transparency in action: search results are turned into virtual attachments so you can inspect which documents contributed to the answer. Click on the attachment button in your prompt to see the search answers!",+ "You can see each tool call with wrench icons (click on it!) in assistant messages and open per-call details (arguments and tool response).",+ "You can use deterministic tools such as datetime, calculator, webfetch, http_json, and table_ops to improve answer quality.",+ "You can copy assistant answers, trim user prompts, and delete a user/assistant turn pair for iterative editing.",+ "You can clear chat history, download the chat as JSON, and upload it again to continue later.",+ "You can show or hide the system prompt in the interface to better understand chat behavior.",+ "You can read markdown-formatted answers, including code blocks with syntax highlighting and collapsible model-thought sections when present.",+ "You can use YaCy AI components beyond chat via standards-oriented integration points such as RAG proxy and MCP tooling."+ ));++ private static final List<String> IDEAS = Collections.unmodifiableList(Arrays.asList(+ "Transparency, not mystery: ranking, retrieval, and model use should be explainable and inspectable.",+ "Autonomy, no dependency: avoid hidden data flows and centralized lock-in whenever possible.",+ "Collaboration with user control: intelligence should grow from shared knowledge while users stay in charge.",+ "Reproducibility: users should be able to choose models and verify outcomes.",+ "Integration instead of isolation: AI should be part of the open web, not a wall around it.",+ "Teach openly: communicate purpose and benefit so AI is understandable, not opaque or intimidating.",+ "YaCy extends open-search principles into AI: openness, autonomy, and trust."+ ));++ private static final String TALKS_URL = "https://www.youtube.com/orbiterlab";++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "This is the help function. Explain what YaCy AI does and why it is built this way. Use when user asks about abilities, purpose, or philosophy.");+ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ params.put("properties", new JSONObject(true));+ params.put("required", new JSONArray());+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }+++ public int maxCallsPerTurn() {+ return 1;+ }+++ public String execute(String arguments) {+ try {+ JSONObject result = new JSONObject(true);+ result.put("tool", NAME);+ result.put("purpose", PURPOSE);+ result.put("facts", new JSONArray(FACTS));+ result.put("ideas", new JSONArray(IDEAS));+ result.put("talks_url", TALKS_URL);+ return result.toString();+ } catch (JSONException e) {+ return ToolHandler.errorJson("Failed to build self_reflect response");+ }+ }++ public static List<String> facts() {+ return FACTS;+ }++ public static List<String> ideas() {+ return IDEAS;+ }++ public static String purpose() {+ return PURPOSE;+ }+}diff --git a/source/net/yacy/ai/tools/TableOpsTool.java b/source/net/yacy/ai/tools/TableOpsTool.javanew file mode 100644index 000000000..fe1b71699--- /dev/null+++ b/source/net/yacy/ai/tools/TableOpsTool.java@@ -0,0 +1,285 @@+/**+ * TableOpsTool+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.util.ArrayList;+import java.util.Collections;+import java.util.Comparator;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.ai.ToolHandler;++public class TableOpsTool implements ToolHandler {++ private static final String NAME = "table_ops";++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "Apply deterministic table operations to a JSON array of row objects.");++ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ JSONObject props = new JSONObject(true);++ JSONObject rows = new JSONObject(true);+ rows.put("type", "array");+ rows.put("description", "Input table rows as an array of objects.");+ props.put("rows", rows);++ JSONObject operations = new JSONObject(true);+ operations.put("type", "array");+ operations.put("description", "Pipeline of operations: filter, sort, limit, project, group_count.");+ props.put("operations", operations);++ params.put("properties", props);+ params.put("required", new JSONArray().put("rows").put("operations"));+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }+++ public int maxCallsPerTurn() {+ return 3;+ }+++ public String execute(String arguments) {+ final JSONObject args;+ try {+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);+ } catch (JSONException e) {+ return ToolHandler.errorJson("Invalid arguments JSON");+ }++ final JSONArray rowsArray = args.optJSONArray("rows");+ final JSONArray ops = args.optJSONArray("operations");+ if (rowsArray == null) return ToolHandler.errorJson("Missing rows");+ if (ops == null) return ToolHandler.errorJson("Missing operations");++ try {+ List<JSONObject> rows = toObjectRows(rowsArray);+ for (int i = 0; i < ops.length(); i++) {+ JSONObject op = ops.optJSONObject(i);+ if (op == null) return ToolHandler.errorJson("Operation at index " + i + " must be an object");+ rows = applyOperation(rows, op);+ }+ JSONObject result = new JSONObject(true);+ result.put("row_count", rows.size());+ result.put("rows", new JSONArray(rows));+ return result.toString();+ } catch (IllegalArgumentException | JSONException e) {+ return ToolHandler.errorJson(e.getMessage());+ }+ }++ private static List<JSONObject> toObjectRows(JSONArray array) {+ List<JSONObject> rows = new ArrayList<>();+ for (int i = 0; i < array.length(); i++) {+ JSONObject row = array.optJSONObject(i);+ if (row == null) throw new IllegalArgumentException("Row at index " + i + " is not an object");+ rows.add(row);+ }+ return rows;+ }++ private static List<JSONObject> applyOperation(List<JSONObject> rows, JSONObject op) {+ String type = op.optString("type", "").trim().toLowerCase();+ if (type.isEmpty()) throw new IllegalArgumentException("Operation missing type");+ if ("filter".equals(type)) return applyFilter(rows, op);+ if ("sort".equals(type)) return applySort(rows, op);+ if ("limit".equals(type)) return applyLimit(rows, op);+ if ("project".equals(type)) return applyProject(rows, op);+ if ("group_count".equals(type)) return applyGroupCount(rows, op);+ throw new IllegalArgumentException("Unsupported operation type: " + type);+ }++ private static List<JSONObject> applyFilter(List<JSONObject> rows, JSONObject op) {+ String field = requiredField(op, "field");+ String cmp = op.optString("op", "eq").trim().toLowerCase();+ Object expected = op.opt("value");+ List<JSONObject> result = new ArrayList<>();+ for (JSONObject row : rows) {+ Object actual = row.opt(field);+ if (matches(actual, cmp, expected)) result.add(row);+ }+ return result;+ }++ private static List<JSONObject> applySort(List<JSONObject> rows, JSONObject op) {+ String field = requiredField(op, "field");+ String order = op.optString("order", "asc").trim().toLowerCase();+ final int direction = "desc".equals(order) ? -1 : 1;++ List<JSONObject> copy = new ArrayList<>(rows);+ Collections.sort(copy, new Comparator<JSONObject>() {+ @Override+ public int compare(JSONObject a, JSONObject b) {+ Object va = a.opt(field);+ Object vb = b.opt(field);+ return direction * compareValues(va, vb);+ }+ });+ return copy;+ }++ private static List<JSONObject> applyLimit(List<JSONObject> rows, JSONObject op) {+ int count = op.optInt("count", -1);+ int offset = op.optInt("offset", 0);+ if (count < 0) throw new IllegalArgumentException("limit.count must be >= 0");+ if (offset < 0) throw new IllegalArgumentException("limit.offset must be >= 0");+ if (offset >= rows.size()) return new ArrayList<>();+ int end = Math.min(rows.size(), offset + count);+ return new ArrayList<>(rows.subList(offset, end));+ }++ private static List<JSONObject> applyProject(List<JSONObject> rows, JSONObject op) {+ JSONArray fields = op.optJSONArray("fields");+ if (fields == null || fields.length() == 0) throw new IllegalArgumentException("project.fields must be a non-empty array");+ List<String> keep = new ArrayList<>();+ for (int i = 0; i < fields.length(); i++) {+ String f = fields.optString(i, "").trim();+ if (!f.isEmpty()) keep.add(f);+ }+ if (keep.isEmpty()) throw new IllegalArgumentException("project.fields must contain at least one field");++ List<JSONObject> out = new ArrayList<>();+ for (JSONObject row : rows) {+ JSONObject projected = new JSONObject(true);+ for (String key : keep) {+ if (row.has(key)) {+ try {+ projected.put(key, row.get(key));+ } catch (JSONException e) {+ throw new IllegalArgumentException("Failed to project field: " + key);+ }+ }+ }+ out.add(projected);+ }+ return out;+ }++ private static List<JSONObject> applyGroupCount(List<JSONObject> rows, JSONObject op) {+ String field = requiredField(op, "field");+ String order = op.optString("order", "desc").trim().toLowerCase();+ final int direction = "asc".equals(order) ? 1 : -1;++ Map<String, Integer> counts = new LinkedHashMap<>();+ for (JSONObject row : rows) {+ Object v = row.opt(field);+ String key = v == null || v == JSONObject.NULL ? "null" : String.valueOf(v);+ Integer cur = counts.get(key);+ counts.put(key, cur == null ? 1 : cur + 1);+ }++ List<JSONObject> grouped = new ArrayList<>();+ for (Map.Entry<String, Integer> entry : counts.entrySet()) {+ try {+ JSONObject g = new JSONObject(true);+ g.put("key", entry.getKey());+ g.put("count", entry.getValue());+ grouped.add(g);+ } catch (JSONException e) {+ throw new IllegalArgumentException("Failed to build group_count result");+ }+ }+ Collections.sort(grouped, new Comparator<JSONObject>() {+ @Override+ public int compare(JSONObject a, JSONObject b) {+ return direction * Integer.compare(a.optInt("count", 0), b.optInt("count", 0));+ }+ });+ return grouped;+ }++ private static String requiredField(JSONObject obj, String field) {+ String value = obj.optString(field, "").trim();+ if (value.isEmpty()) throw new IllegalArgumentException("Missing " + field);+ return value;+ }++ private static boolean matches(Object actual, String cmp, Object expected) {+ if ("eq".equals(cmp)) return valuesEqual(actual, expected);+ if ("ne".equals(cmp)) return !valuesEqual(actual, expected);+ if ("gt".equals(cmp)) return compareValues(actual, expected) > 0;+ if ("gte".equals(cmp)) return compareValues(actual, expected) >= 0;+ if ("lt".equals(cmp)) return compareValues(actual, expected) < 0;+ if ("lte".equals(cmp)) return compareValues(actual, expected) <= 0;+ if ("contains".equals(cmp)) {+ if (actual == null || actual == JSONObject.NULL || expected == null || expected == JSONObject.NULL) return false;+ if (actual instanceof JSONArray) return arrayContains((JSONArray) actual, expected);+ return String.valueOf(actual).toLowerCase().contains(String.valueOf(expected).toLowerCase());+ }+ if ("in".equals(cmp)) {+ if (!(expected instanceof JSONArray)) return false;+ return arrayContains((JSONArray) expected, actual);+ }+ throw new IllegalArgumentException("Unsupported filter op: " + cmp);+ }++ private static boolean valuesEqual(Object a, Object b) {+ if (a == b) return true;+ if (a == null || a == JSONObject.NULL) return b == null || b == JSONObject.NULL;+ if (b == null || b == JSONObject.NULL) return false;+ Double na = asDouble(a);+ Double nb = asDouble(b);+ if (na != null && nb != null) return Double.compare(na, nb) == 0;+ return String.valueOf(a).equals(String.valueOf(b));+ }++ private static int compareValues(Object a, Object b) {+ if (a == b) return 0;+ if (a == null || a == JSONObject.NULL) return -1;+ if (b == null || b == JSONObject.NULL) return 1;+ Double na = asDouble(a);+ Double nb = asDouble(b);+ if (na != null && nb != null) return Double.compare(na, nb);+ return String.valueOf(a).compareToIgnoreCase(String.valueOf(b));+ }++ private static Double asDouble(Object value) {+ if (value == null || value == JSONObject.NULL) return null;+ if (value instanceof Number) return Double.valueOf(((Number) value).doubleValue());+ try {+ return Double.valueOf(String.valueOf(value));+ } catch (NumberFormatException e) {+ return null;+ }+ }++ private static boolean arrayContains(JSONArray array, Object wanted) {+ for (int i = 0; i < array.length(); i++) {+ if (valuesEqual(array.opt(i), wanted)) return true;+ }+ return false;+ }+}diff --git a/source/net/yacy/ai/tools/UnitConverterTool.java b/source/net/yacy/ai/tools/UnitConverterTool.javanew file mode 100644index 000000000..dfa41d100--- /dev/null+++ b/source/net/yacy/ai/tools/UnitConverterTool.java@@ -0,0 +1,200 @@+/**+ * UnitConverterTool+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.util.HashMap;+import java.util.Locale;+import java.util.Map;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.ai.ToolHandler;++public class UnitConverterTool implements ToolHandler {++ private static final String NAME = "unit_converter";++ private static final Map<String, Double> LENGTH_TO_M = new HashMap<>();+ private static final Map<String, Double> MASS_TO_KG = new HashMap<>();+ private static final Map<String, Double> VOLUME_TO_L = new HashMap<>();+ private static final Map<String, Double> SPEED_TO_MPS = new HashMap<>();++ static {+ LENGTH_TO_M.put("mm", 0.001d);+ LENGTH_TO_M.put("cm", 0.01d);+ LENGTH_TO_M.put("m", 1d);+ LENGTH_TO_M.put("km", 1000d);+ LENGTH_TO_M.put("in", 0.0254d);+ LENGTH_TO_M.put("ft", 0.3048d);+ LENGTH_TO_M.put("yd", 0.9144d);+ LENGTH_TO_M.put("mi", 1609.344d);++ MASS_TO_KG.put("mg", 0.000001d);+ MASS_TO_KG.put("g", 0.001d);+ MASS_TO_KG.put("kg", 1d);+ MASS_TO_KG.put("t", 1000d);+ MASS_TO_KG.put("oz", 0.028349523125d);+ MASS_TO_KG.put("lb", 0.45359237d);++ VOLUME_TO_L.put("ml", 0.001d);+ VOLUME_TO_L.put("l", 1d);+ VOLUME_TO_L.put("m3", 1000d);+ VOLUME_TO_L.put("tsp", 0.00492892159375d);+ VOLUME_TO_L.put("tbsp", 0.01478676478125d);+ VOLUME_TO_L.put("cup", 0.2365882365d);+ VOLUME_TO_L.put("pt", 0.473176473d);+ VOLUME_TO_L.put("qt", 0.946352946d);+ VOLUME_TO_L.put("gal", 3.785411784d);++ SPEED_TO_MPS.put("m/s", 1d);+ SPEED_TO_MPS.put("km/h", 0.2777777777777778d);+ SPEED_TO_MPS.put("mph", 0.44704d);+ SPEED_TO_MPS.put("kn", 0.514444d);+ }++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "Convert numeric values between common units (length, mass, temperature, volume, speed).");+ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ JSONObject props = new JSONObject(true);+ props.put("value", new JSONObject(true).put("type", "number").put("description", "Numeric input value."));+ props.put("from", new JSONObject(true).put("type", "string").put("description", "Source unit, e.g. km, lb, C."));+ props.put("to", new JSONObject(true).put("type", "string").put("description", "Target unit, e.g. mi, kg, F."));+ params.put("properties", props);+ params.put("required", new JSONArray().put("value").put("from").put("to"));+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }+++ public int maxCallsPerTurn() {+ return 10;+ }+++ public String execute(String arguments) {+ final JSONObject args;+ try {+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);+ } catch (JSONException e) {+ return ToolHandler.errorJson("Invalid arguments JSON");+ }+ if (!args.has("value")) return ToolHandler.errorJson("Missing value");+ double value = args.optDouble("value", Double.NaN);+ if (Double.isNaN(value) || Double.isInfinite(value)) return ToolHandler.errorJson("Invalid value");+ String from = normalize(args.optString("from", ""));+ String to = normalize(args.optString("to", ""));+ if (from.isEmpty() || to.isEmpty()) return ToolHandler.errorJson("Missing from/to");++ try {+ ConversionResult result = convert(value, from, to);+ JSONObject out = new JSONObject(true);+ out.put("tool", NAME);+ out.put("category", result.category);+ out.put("value", value);+ out.put("from", from);+ out.put("to", to);+ out.put("result", result.value);+ return out.toString();+ } catch (IllegalArgumentException | JSONException e) {+ return ToolHandler.errorJson(e.getMessage());+ }+ }++ private static ConversionResult convert(double value, String from, String to) {+ if (isTemp(from) && isTemp(to)) {+ double c = toCelsius(value, from);+ return new ConversionResult("temperature", fromCelsius(c, to));+ }+ if (LENGTH_TO_M.containsKey(from) && LENGTH_TO_M.containsKey(to)) {+ double m = value * LENGTH_TO_M.get(from);+ return new ConversionResult("length", m / LENGTH_TO_M.get(to));+ }+ if (MASS_TO_KG.containsKey(from) && MASS_TO_KG.containsKey(to)) {+ double kg = value * MASS_TO_KG.get(from);+ return new ConversionResult("mass", kg / MASS_TO_KG.get(to));+ }+ if (VOLUME_TO_L.containsKey(from) && VOLUME_TO_L.containsKey(to)) {+ double l = value * VOLUME_TO_L.get(from);+ return new ConversionResult("volume", l / VOLUME_TO_L.get(to));+ }+ if (SPEED_TO_MPS.containsKey(from) && SPEED_TO_MPS.containsKey(to)) {+ double mps = value * SPEED_TO_MPS.get(from);+ return new ConversionResult("speed", mps / SPEED_TO_MPS.get(to));+ }+ throw new IllegalArgumentException("Unsupported conversion pair: " + from + " -> " + to);+ }++ private static boolean isTemp(String unit) {+ return "c".equals(unit) || "f".equals(unit) || "k".equals(unit);+ }++ private static double toCelsius(double v, String from) {+ if ("c".equals(from)) return v;+ if ("f".equals(from)) return (v - 32d) * 5d / 9d;+ if ("k".equals(from)) return v - 273.15d;+ throw new IllegalArgumentException("Unsupported temperature unit: " + from);+ }++ private static double fromCelsius(double c, String to) {+ if ("c".equals(to)) return c;+ if ("f".equals(to)) return (c * 9d / 5d) + 32d;+ if ("k".equals(to)) return c + 273.15d;+ throw new IllegalArgumentException("Unsupported temperature unit: " + to);+ }++ private static String normalize(String s) {+ if (s == null) return "";+ String t = s.trim().toLowerCase(Locale.ROOT);+ if ("°c".equals(t) || "celsius".equals(t)) return "c";+ if ("°f".equals(t) || "fahrenheit".equals(t)) return "f";+ if ("kelvin".equals(t)) return "k";+ if ("meter".equals(t) || "metre".equals(t)) return "m";+ if ("kilometer".equals(t) || "kilometre".equals(t)) return "km";+ if ("mile".equals(t)) return "mi";+ if ("foot".equals(t)) return "ft";+ if ("inch".equals(t)) return "in";+ if ("pound".equals(t)) return "lb";+ if ("ounce".equals(t)) return "oz";+ if ("liter".equals(t) || "litre".equals(t)) return "l";+ if ("gallon".equals(t)) return "gal";+ if ("kph".equals(t)) return "km/h";+ if ("fps".equals(t)) return "ft/s";+ return t;+ }++ private static final class ConversionResult {+ final String category;+ final double value;+ ConversionResult(String category, double value) {+ this.category = category;+ this.value = value;+ }+ }+}diff --git a/source/net/yacy/ai/tools/WebFetchTool.java b/source/net/yacy/ai/tools/WebFetchTool.javanew file mode 100644index 000000000..e09669807--- /dev/null+++ b/source/net/yacy/ai/tools/WebFetchTool.java@@ -0,0 +1,168 @@+/**+ * WebFetchTool+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.util.Date;+import java.util.HashSet;++import org.apache.http.Header;+import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.ai.ToolHandler;+import net.yacy.cora.document.analysis.Classification;+import net.yacy.cora.document.analysis.Classification.ContentDomain;+import net.yacy.cora.document.id.DigestURL;+import net.yacy.cora.document.id.MultiProtocolURL;+import net.yacy.cora.protocol.ClientIdentification;+import net.yacy.cora.protocol.HeaderFramework;+import net.yacy.cora.protocol.http.HTTPClient;+import net.yacy.document.Document;+import net.yacy.document.Parser;+import net.yacy.document.TextParser;+import net.yacy.document.VocabularyScraper;+import net.yacy.document.parser.html.TagValency;++public class WebFetchTool implements ToolHandler {++ private static final String NAME = "webfetch";+ private static final int TOOL_WEBFETCH_MAX_BYTES = 2_000_000;+ private static final int TOOL_WEBFETCH_MAX_CHARS = 12_000;++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "Fetch a URL and return text content. Parse HTML/documents into markdown-style text when possible.");+ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ JSONObject props = new JSONObject(true);+ JSONObject url = new JSONObject(true);+ url.put("type", "string");+ url.put("description", "Absolute http or https URL to fetch.");+ props.put("url", url);+ params.put("properties", props);+ params.put("required", new JSONArray().put("url"));+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }+++ public int maxCallsPerTurn() {+ return 3;+ }+++ public String execute(String arguments) {+ String urlRaw;+ try {+ JSONObject obj = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);+ urlRaw = obj.optString("url", "").trim();+ } catch (JSONException e) {+ return ToolHandler.errorJson("Invalid arguments JSON. don't try again");+ }+ if (urlRaw == null || urlRaw.isEmpty()) return ToolHandler.errorJson("Missing url. don't try again");++ final DigestURL url;+ try {+ url = new DigestURL(urlRaw);+ } catch (Exception e) {+ return ToolHandler.errorJson("Invalid URL. don't try again");+ }+ final String protocol = url.getProtocol();+ if (!"http".equalsIgnoreCase(protocol) && !"https".equalsIgnoreCase(protocol)) {+ return ToolHandler.errorJson("Only http/https URLs are allowed. don't try again");+ }++ try (HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000)) {+ byte[] content = client.GETbytes(url, null, null, TOOL_WEBFETCH_MAX_BYTES, false);+ int status = client.getStatusCode();+ if (status < 200 || status >= 300) return ToolHandler.errorJson("HTTP status " + status + ". don't try again");+ if (content == null || content.length == 0) return ToolHandler.errorJson("Empty response body. don't try again");++ String contentType = "application/octet-stream";+ Header ct = client.getHttpResponse() == null ? null : client.getHttpResponse().getFirstHeader(HeaderFramework.CONTENT_TYPE);+ if (ct != null && ct.getValue() != null && !ct.getValue().isEmpty()) contentType = ct.getValue();+ int semicolon = contentType.indexOf(';');+ if (semicolon > 0) contentType = contentType.substring(0, semicolon).trim().toLowerCase();++ final String ext = MultiProtocolURL.getFileExtension(url.getFileName());+ final ContentDomain mimeDomain = Classification.getContentDomainFromMime(contentType);+ if (mimeDomain == ContentDomain.IMAGE || mimeDomain == ContentDomain.AUDIO || mimeDomain == ContentDomain.VIDEO+ || Classification.isMediaExtension(ext)) {+ return ToolHandler.errorJson("Media content is not supported. don't try again");+ }++ String text;+ if (contentType.startsWith("text/plain") || contentType.startsWith("text/markdown") || "txt".equals(ext) || "md".equals(ext)) {+ text = truncate(new String(content, StandardCharsets.UTF_8), TOOL_WEBFETCH_MAX_CHARS);+ } else {+ String supportError = TextParser.supports(url, contentType);+ if (supportError == null) {+ Document[] docs = TextParser.parseSource(url, contentType, "UTF-8", TagValency.EVAL,+ new HashSet<String>(), new VocabularyScraper(), 0, 0, content, new Date());+ Document merged = Document.mergeDocuments(url, contentType, docs);+ text = truncate(documentToMarkdown(merged), TOOL_WEBFETCH_MAX_CHARS);+ } else if (contentType.startsWith("text/")) {+ text = truncate(new String(content, StandardCharsets.UTF_8), TOOL_WEBFETCH_MAX_CHARS);+ } else {+ return ToolHandler.errorJson("Unsupported content type " + contentType + ". don't try again");+ }+ }++ JSONObject result = new JSONObject(true);+ result.put("url", url.toNormalform(true));+ result.put("content_type", contentType);+ result.put("content", text == null ? "" : text);+ return result.toString();+ } catch (IOException e) {+ return ToolHandler.errorJson("Fetch error: " + e.getMessage() + ". don't try again");+ } catch (Parser.Failure e) {+ return ToolHandler.errorJson("Parse error: " + e.getMessage() + ". don't try again");+ } catch (JSONException e) {+ return ToolHandler.errorJson("Failed to build tool response. don't try again");+ }+ }++ private static String documentToMarkdown(Document doc) {+ if (doc == null) return "";+ StringBuilder sb = new StringBuilder();+ String title = doc.dc_title();+ if (title != null && !title.isEmpty()) {+ sb.append("# ").append(title).append("\n\n");+ }+ String text = doc.getTextString();+ if (text != null && !text.isEmpty()) sb.append(text);+ return sb.toString();+ }++ private static String truncate(String text, int maxChars) {+ if (text == null) return "";+ if (maxChars <= 0 || text.length() <= maxChars) return text;+ return text.substring(0, maxChars);+ }+}diff --git a/source/net/yacy/ai/tools/WikipediaLinkCreatorTool.java b/source/net/yacy/ai/tools/WikipediaLinkCreatorTool.javanew file mode 100644index 000000000..977d9574d--- /dev/null+++ b/source/net/yacy/ai/tools/WikipediaLinkCreatorTool.java@@ -0,0 +1,169 @@+/**+ * WikipediaLinkCreatorTool+ * Copyright 2026 by Michael Peter Christen+ * First released 06.02.2026 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.ai.tools;++import java.io.UnsupportedEncodingException;+import java.net.URLEncoder;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.List;+import java.util.Locale;+import java.util.regex.Pattern;++import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;++import net.yacy.ai.ToolHandler;++public class WikipediaLinkCreatorTool implements ToolHandler {++ private static final String NAME = "wikipedia_link_creator";+ private static final Pattern LANG_PATTERN = Pattern.compile("^[a-z]{2,10}(-[a-z0-9]{2,8})?$");++ @Override+ public JSONObject definition() throws JSONException {+ JSONObject tool = new JSONObject(true);+ tool.put("type", "function");+ JSONObject fn = new JSONObject(true);+ fn.put("name", NAME);+ fn.put("description", "In case you don't know the answer correctly or you are unsure. Build a Wikipedia article URL from topic keywords so it can be fetched with webfetch.");++ JSONObject params = new JSONObject(true);+ params.put("type", "object");+ JSONObject props = new JSONObject(true);++ JSONObject topic = new JSONObject(true);+ topic.put("type", "string");+ topic.put("description", "Target Wikipedia article title/topic (preferred input).");+ props.put("topic", topic);++ JSONObject keywords = new JSONObject(true);+ keywords.put("type", "array");+ keywords.put("description", "Optional keyword list from which a topic is composed.");+ keywords.put("items", new JSONObject(true).put("type", "string"));+ props.put("keywords", keywords);++ JSONObject language = new JSONObject(true);+ language.put("type", "string");+ language.put("description", "Wikipedia language code (e.g. en, de, fr). Defaults to en.");+ props.put("language", language);++ params.put("properties", props);+ params.put("required", new JSONArray());+ fn.put("parameters", params);+ tool.put("function", fn);+ return tool;+ }+++ public int maxCallsPerTurn() {+ return 3;+ }+++ public String execute(String arguments) {+ final JSONObject args;+ try {+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);+ } catch (JSONException e) {+ return ToolHandler.errorJson("Invalid arguments JSON");+ }++ String topic = normalizeTopic(args.optString("topic", ""));+ JSONArray keywords = args.optJSONArray("keywords");+ String language = normalizeLanguage(args.optString("language", "en"));+ List<String> warnings = new ArrayList<>();++ if (topic.isEmpty() && keywords != null) {+ topic = normalizeTopic(joinKeywords(keywords));+ if (!topic.isEmpty()) {+ warnings.add("Used keywords to compose article topic");+ }+ }+ if (topic.isEmpty()) {+ return ToolHandler.errorJson("Missing topic/keywords");+ }+ if (language == null) {+ language = "en";+ warnings.add("Invalid language code; defaulted to en");+ }++ String encodedTitle = encodeTitle(topic);+ String url = "https://" + language + ".wikipedia.org/wiki/" + encodedTitle;++ try {+ JSONObject result = new JSONObject(true);+ result.put("tool", NAME);+ result.put("topic_input", topic);+ result.put("article_title", topic);+ result.put("language", language);+ result.put("url", url);+ result.put("next_tool_hint", "Use webfetch with this URL to retrieve the article content.");+ result.put("warnings", new JSONArray(warnings));+ return result.toString();+ } catch (JSONException e) {+ return ToolHandler.errorJson("Failed to build wikipedia link response");+ }+ }++ private static String joinKeywords(JSONArray keywords) {+ StringBuilder sb = new StringBuilder();+ for (int i = 0; i < keywords.length(); i++) {+ String k = keywords.optString(i, "").trim();+ if (k.isEmpty()) continue;+ if (sb.length() > 0) sb.append(' ');+ sb.append(k);+ }+ return sb.toString();+ }++ private static String normalizeTopic(String topic) {+ if (topic == null) return "";+ String t = topic.trim();+ while (t.contains(" ")) t = t.replace(" ", " ");+ if (t.startsWith("https://") || t.startsWith("http://")) {+ int wikiIdx = t.indexOf("/wiki/");+ if (wikiIdx >= 0) {+ t = t.substring(wikiIdx + 6);+ }+ }+ t = t.replace('_', ' ').trim();+ if (t.indexOf('#') >= 0) t = t.substring(0, t.indexOf('#')).trim();+ return t;+ }++ private static String normalizeLanguage(String language) {+ if (language == null) return "en";+ String l = language.trim().toLowerCase(Locale.ROOT);+ if (l.isEmpty()) return "en";+ if (!LANG_PATTERN.matcher(l).matches()) return null;+ return l;+ }++ private static String encodeTitle(String title) {+ try {+ return URLEncoder.encode(title, StandardCharsets.UTF_8.name()).replace("+", "_");+ } catch (UnsupportedEncodingException e) {+ return title.replace(' ', '_');+ }+ }+}diff --git a/source/net/yacy/http/servlets/MCPSearchServlet.java b/source/net/yacy/http/servlets/MCPSearchServlet.javaindex ffca72aac..983ad24f7 100644--- a/source/net/yacy/http/servlets/MCPSearchServlet.java+++ b/source/net/yacy/http/servlets/MCPSearchServlet.java@@ -20,6 +20,7 @@package net.yacy.http.servlets;+import net.yacy.ai.RAGAugmentor;import net.yacy.cora.protocol.HeaderFramework;import net.yacy.cora.util.ConcurrentLog;@@ -58,6 +59,17 @@ public class MCPSearchServlet extends HttpServlet {private static final int DEFAULT_RESULT_COUNT = 10;private static final int MAX_RESULT_COUNT = 100;+ /**+ * Handles JSON-RPC requests for the MCP surface.+ * <p>+ * Accepts single request objects and batch arrays, supports notifications, and+ * always returns UTF-8 JSON responses.+ *+ * @param request servlet request+ * @param response servlet response+ * @throws ServletException servlet errors+ * @throws IOException I/O errors+ */@Overridepublic void service(final ServletRequest request, final ServletResponse response) throws ServletException, IOException {request.setCharacterEncoding("UTF-8");@@ -72,6 +84,7 @@ public class MCPSearchServlet extends HttpServlet {final Method reqMethod = Method.getMethod(hrequest.getMethod());if (reqMethod == Method.OTHER) {+ // Browsers perform CORS preflight with OPTIONS; answer quickly.hresponse.setStatus(HttpServletResponse.SC_OK);return;}@@ -85,10 +98,12 @@ public class MCPSearchServlet extends HttpServlet {parsed = tokener.nextValue();} catch (JSONException e) {writeJsonResponse(hresponse, errorResponse(JSONObject.NULL, -32700, e.getMessage()));+ return;}try {if (parsed instanceof JSONObject) {+ // Support shorthand endpoint style: infer method from URI path.if (((JSONObject) parsed).optString("method", "").length() == 0) {String uri = hrequest.getRequestURI();((JSONObject) parsed).put("method", uri.substring(1));@@ -100,6 +115,7 @@ public class MCPSearchServlet extends HttpServlet {hresponse.setStatus(HttpServletResponse.SC_NO_CONTENT);}} else if (parsed instanceof JSONArray) {+ // Batch request mode.final JSONArray requestArray = (JSONArray) parsed;final JSONArray responseArray = new JSONArray();for (int i = 0; i < requestArray.length(); i++) {@@ -124,6 +140,12 @@ public class MCPSearchServlet extends HttpServlet {}}+ /**+ * Routes one JSON-RPC request object to the corresponding MCP handler.+ *+ * @param requestObject parsed request+ * @return response object or {@code null} for notifications+ */private JSONObject handleRequest(final JSONObject requestObject) {final Object id = requestObject.opt("id");final String jsonrpc = requestObject.optString("jsonrpc", JSONRPC_VERSION);@@ -147,6 +169,7 @@ public class MCPSearchServlet extends HttpServlet {}if (id == JSONObject.NULL || id == null) {+ // Notifications must not produce a response body.// Notification: acknowledge silentlyif ("ping".equals(method)) {return null;@@ -167,6 +190,13 @@ public class MCPSearchServlet extends HttpServlet {}}+ /**+ * Handles MCP initialize call and advertises protocol/capabilities.+ *+ * @param id JSON-RPC id+ * @param params initialize params+ * @return JSON-RPC success or error response+ */private JSONObject handleInitialize(final Object id, final JSONObject params) {try {log.info("MCPSearchServlet: initialize " + (params == null ? "" : params.toString(0)));} catch (JSONException e) {}final JSONObject result = new JSONObject(true);@@ -190,6 +220,13 @@ public class MCPSearchServlet extends HttpServlet {return successResponse(id, result);}+ /**+ * Handles MCP tool listing and returns schema for the single {@code search}+ * tool.+ *+ * @param id JSON-RPC id+ * @return JSON-RPC success or error response+ */private JSONObject handleToolsList(final Object id) {log.info("MCPSearchServlet: list " + (id == null ? "" : id.toString()));try {@@ -245,6 +282,13 @@ public class MCPSearchServlet extends HttpServlet {}}+ /**+ * Handles tool invocation for {@code search}.+ *+ * @param id JSON-RPC id+ * @param params tool call params+ * @return JSON-RPC success or error response+ */private JSONObject handleToolsCall(final Object id, final JSONObject params) {try {log.info("MCPSearchServlet: call " + (id == null ? "" : id.toString()) + " params: " + (params == null ? "" : params.toString(0)));} catch (JSONException e) {}if (params == null) {@@ -265,6 +309,7 @@ public class MCPSearchServlet extends HttpServlet {return errorResponse(id, -32602, "Tool argument 'query' must be a non-empty string");}+ // Sanitize and clamp result limit.int limit = arguments.optInt("limit", DEFAULT_RESULT_COUNT);if (limit <= 0) {limit = DEFAULT_RESULT_COUNT;@@ -272,8 +317,9 @@ public class MCPSearchServlet extends HttpServlet {limit = Math.min(limit, MAX_RESULT_COUNT);final boolean includeSnippet = arguments.optBoolean("include_snippet", true);+ // Delegate the actual search to the RAG helper utility.JSONArray results;- results = RAGProxyServlet.searchResults(query, limit, includeSnippet);+ results = RAGAugmentor.searchResults(query, limit, includeSnippet);try {final JSONObject payload = new JSONObject(true);@@ -294,6 +340,13 @@ public class MCPSearchServlet extends HttpServlet {}}+ /**+ * Reads full HTTP request body into a string.+ *+ * @param request servlet request+ * @return request body (possibly empty)+ * @throws IOException when reader access fails+ */private static String readBody(final ServletRequest request) throws IOException {final StringBuilder builder = new StringBuilder();try (BufferedReader reader = request.getReader()) {@@ -305,6 +358,13 @@ public class MCPSearchServlet extends HttpServlet {return builder.toString();}+ /**+ * Builds a JSON-RPC success object.+ *+ * @param id JSON-RPC id+ * @param result result payload+ * @return JSON-RPC success response+ */private static JSONObject successResponse(final Object id, final JSONObject result) {final JSONObject response = new JSONObject(true);try {@@ -318,6 +378,14 @@ public class MCPSearchServlet extends HttpServlet {return response;}+ /**+ * Builds a JSON-RPC error object.+ *+ * @param id JSON-RPC id+ * @param code JSON-RPC error code+ * @param message human-readable error message+ * @return JSON-RPC error response+ */private static JSONObject errorResponse(final Object id, final int code, final String message) {final JSONObject response = new JSONObject(true);try {@@ -333,6 +401,13 @@ public class MCPSearchServlet extends HttpServlet {return response;}+ /**+ * Serializes and writes either a JSON object or JSON array response.+ *+ * @param response servlet response+ * @param payload JSONObject or JSONArray+ * @throws IOException when writing fails+ */private static void writeJsonResponse(final HttpServletResponse response, final Object payload) throws IOException {final String serialized = payload instanceof JSONObject ? ((JSONObject) payload).toString(): payload instanceof JSONArray ? ((JSONArray) payload).toString() : payload.toString();diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.javaindex 840e5a564..c552b7387 100644--- a/source/net/yacy/http/servlets/RAGProxyServlet.java+++ b/source/net/yacy/http/servlets/RAGProxyServlet.java@@ -22,31 +22,14 @@ package net.yacy.http.servlets;import java.io.BufferedReader;import java.io.IOException;-import java.io.InputStreamReader;-import java.io.OutputStream;-import java.net.HttpURLConnection;-import java.net.URI;-import java.net.URISyntaxException;-import java.net.URL;import java.nio.charset.StandardCharsets;import java.util.AbstractMap;import java.util.ArrayList;-import java.util.Arrays;import java.util.Base64;-import java.util.Collection;-import java.util.Comparator;import java.util.Deque;-import java.util.HashMap;-import java.util.HashSet;-import java.util.Iterator;-import java.util.LinkedHashSet;import java.util.List;-import java.util.Map;import java.util.Set;-import java.util.concurrent.BlockingQueue;-import java.util.concurrent.LinkedBlockingQueue;import java.util.concurrent.ConcurrentLinkedDeque;-import java.util.stream.Collectors;import javax.servlet.ServletException;import javax.servlet.ServletOutputStream;@@ -56,38 +39,17 @@ import javax.servlet.http.HttpServlet;import javax.servlet.http.HttpServletRequest;import javax.servlet.http.HttpServletResponse;-import org.apache.solr.client.solrj.SolrQuery;-import org.apache.solr.common.SolrDocument;-import org.apache.solr.common.SolrDocumentList;-import org.apache.solr.common.SolrException;import org.apache.solr.servlet.cache.Method;import org.json.JSONArray;import org.json.JSONException;import org.json.JSONObject;-import org.json.JSONTokener;import net.yacy.ai.LLM;-import net.yacy.cora.document.analysis.Classification;-import net.yacy.cora.document.id.DigestURL;-import net.yacy.cora.document.id.MultiProtocolURL;-import net.yacy.cora.federate.solr.SolrType;-import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;-import net.yacy.cora.federate.yacy.CacheStrategy;-import net.yacy.cora.lod.vocabulary.Tagging;-import net.yacy.cora.protocol.ClientIdentification;+import net.yacy.ai.RAGAugmentor;+import net.yacy.ai.ToolCallProtocol;import net.yacy.cora.protocol.Domains;import net.yacy.cora.util.ConcurrentLog;-import net.yacy.kelondro.data.meta.URIMetadataNode;import net.yacy.search.Switchboard;-import net.yacy.search.SwitchboardConstants;-import net.yacy.search.query.QueryGoal;-import net.yacy.search.query.QueryModifier;-import net.yacy.search.query.QueryParams;-import net.yacy.search.query.SearchEvent;-import net.yacy.search.query.SearchEventCache;-import net.yacy.search.ranking.RankingProfile;-import net.yacy.search.schema.CollectionSchema;-import net.yacy.search.snippet.TextSnippet;/*** This class implements a Retrieval Augmented Generation ("RAG") proxy which@@ -109,7 +71,6 @@ public class RAGProxyServlet extends HttpServlet {private static final long serialVersionUID = 3411544789759643137L;public static final String LLM_SYSTEM_PROMPT_DEFAULT = "You are a smart and helpful chatbot. If possible, use friendly emojies.";- private static final String LLM_SYSTEM_PREFIX_DEFAULT = "\n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.";private static final String LLM_USER_PREFIX_DEFAULT = "\n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n";private static final String LLM_QUERY_GENERATOR_PREFIX_DEFAULT = "Make a list of search words with low document frequency for the following prompt; use a JSON Array: ";@@ -193,7 +154,6 @@ public class RAGProxyServlet extends HttpServlet {// get messages and prepare user message attachmentsJSONArray messages = bodyObject.optJSONArray("messages");- final String systemPrefix = sb.getConfig("ai.llm-system-prefix", LLM_SYSTEM_PREFIX_DEFAULT);final String userPrefix = sb.getConfig("ai.llm-user-prefix", LLM_USER_PREFIX_DEFAULT);// debug@@ -206,26 +166,40 @@ public class RAGProxyServlet extends HttpServlet {userObject.attachAttachment(userPrefix);}}- UserObject userObject = new UserObject(messages.getJSONObject(messages.length() - 1));- String user = userObject.getContentText(); // this is the latest prompt- final String userPrompt = user;- String ragMode = userObject.getSearchMode();+ UserObject userObject = null;+ String user = "";+ String ragMode = "no";+ String userPrompt = "";+ int lastUserIndex = -1;+ for (int i = messages.length() - 1; i >= 0; i--) {+ JSONObject message = messages.getJSONObject(i);+ if ("user".equals(message.optString("role", ""))) {+ lastUserIndex = i;+ break;+ }+ }+ if (lastUserIndex >= 0) {+ userObject = new UserObject(messages.getJSONObject(lastUserIndex));+ user = userObject.getContentText(); // this is the latest user prompt+ userPrompt = user;+ ragMode = userObject.getSearchMode();+ }ConcurrentLog.info("RAGProxy", "ragMode=" + ragMode + " userChars=" + (user == null ? 0 : user.length()));//List<DataURL> data_urls = userObject.getContentAttachments(); // this list is a copy of the content data_urls// RAGString searchResultQuery = "";String searchResultMarkdown = "";- if (!"no".equals(ragMode)) {+ if (userObject != null && !"no".equals(ragMode)) {// modify system and user prompt here in bodyObject to enable RAGfinal String queryPrefix = sb.getConfig("ai.llm-query-generator-prefix", LLM_QUERY_GENERATOR_PREFIX_DEFAULT);final long queryStart = System.currentTimeMillis();- searchResultQuery = this.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, queryPrefix + user); // might return null in case any error occurred+ searchResultQuery = RAGAugmentor.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, queryPrefix + user); // might return null in case any error occurredif (searchResultQuery == null || searchResultQuery.length() == 0) searchResultQuery = user; // in case there is an error we simply search with the promptfinal long queryElapsed = System.currentTimeMillis() - queryStart;- final Set<String> boostTerms = intersectTokens(userPrompt, searchResultQuery, 8);+ final Set<String> boostTerms = RAGAugmentor.intersectTokens(userPrompt, searchResultQuery, 8);final long searchStart = System.currentTimeMillis();- searchResultMarkdown = searchResultsAsMarkdown(searchResultQuery, 10, "global".equals(ragMode), boostTerms);+ searchResultMarkdown = RAGAugmentor.searchResultsAsMarkdown(searchResultQuery, 10, "global".equals(ragMode), boostTerms);final long searchElapsed = System.currentTimeMillis() - searchStart;ConcurrentLog.info("RAGProxy",@@ -236,76 +210,22 @@ public class RAGProxyServlet extends HttpServlet {userObject.setContentText(user);}- // write back modified bodyMap to body- body = bodyObject.toString();-- // Open request to back-end service- final URL url = new URI(llm4Chat.llm.hoststub + "/v1/chat/completions").toURL();- final HttpURLConnection conn = (HttpURLConnection) url.openConnection();- conn.setRequestMethod("POST");- conn.setRequestProperty("Content-Type", "application/json");- if (!llm4Chat.llm.api_key.isEmpty()) {- conn.setRequestProperty("Authorization", "Bearer " + llm4Chat.llm.api_key);+ JSONObject initialMetadata = null;+ if (searchResultMarkdown.length() > 0) {+ initialMetadata = new JSONObject(true);+ initialMetadata.put("search-filename", "search_result_" + searchResultQuery.replace(' ', '_') + ".md");+ initialMetadata.put("search-text-base64", new String(Base64.getEncoder().encode(searchResultMarkdown.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8));}- conn.setDoOutput(true);-- // write the body to back-end LLM- try (OutputStream os = conn.getOutputStream()) {- os.write(body.getBytes());- os.flush();- } // here we wait for the response from upstream- // write back response of the back-end service to the client; use status of- // backend-response- final int status = conn.getResponseCode();- // String rmessage = conn.getResponseMessage();+ // ToolCallProtocol owns request preparation, initial stream handling and follow-up tool rounds.+ final int status = ToolCallProtocol.proxyToolLifecycle(out, llm4Chat, bodyObject, messages, initialMetadata);hresponse.setStatus(status);-- if (status == 200) {- final BlockingQueue<String> inputQueue = new LinkedBlockingQueue<>();- final String POISON = "POISON";- Thread readerThread = new Thread(() -> {- // read the response of the back-end line-by-line and push it to a stack concurrently- try {- final BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));- String inputLine;- while ((inputLine = in.readLine()) != null) {inputQueue.put(inputLine);}- in.close();- inputQueue.put(POISON);- } catch (IOException | InterruptedException e) {- } finally {- try {inputQueue.put(POISON);} catch (InterruptedException e) {}- }- });- readerThread.start();-- // read the stack line-by-line and write it to the client line-by-line- try {- String inputLine;- int count = 0;- while (!(inputLine = inputQueue.take()).equals(POISON)) {- if (count == 0 && searchResultMarkdown.length() > 0) {- // for the first line we modify the data line to integrate the search result as file- int p = inputLine.indexOf('{');- if (p > 0) {- JSONObject j = new JSONObject(new JSONTokener(inputLine.substring(p)));- j.put("search-filename", "search_result_"+ searchResultQuery.replace(' ', '_') + ".md");- j.put("search-text-base64", new String(Base64.getEncoder().encode(searchResultMarkdown.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8));- inputLine = inputLine.substring(0, p) + j.toString();- }- }- out.println(inputLine); // i.e. data: {"id":"chatcmpl-69","object":"chat.completion.chunk","created":1715908287,"model":"llama3:8b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"ߘ"},"finish_reason":null}]}- out.flush();- count++;- }- } catch (InterruptedException e) {}- }out.close(); // close this here to end transmission- } catch (JSONException | URISyntaxException e) {+ } catch (JSONException e) {throw new IOException(e.getMessage());}}-+public final static class DataURL {private String mimetype;private byte[] data;@@ -482,495 +402,6 @@ public class RAGProxyServlet extends HttpServlet {}}}-- public static JSONArray searchResults(String query, int count, final boolean includeSnippet) {- return searchResults(query, count, includeSnippet, new LinkedHashSet<>());- }-- public static JSONArray searchResults(String query, int count, final boolean includeSnippet, final Set<String> boostTerms) {- final JSONArray results = new JSONArray();- if (query == null || query.length() == 0 || count == 0) return results;- Switchboard sb = Switchboard.getSwitchboard();- EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector();- // construct query- final SolrQuery params = new SolrQuery();- params.setQuery(query);- params.set("defType", "edismax");- params.set("qf",- CollectionSchema.title.getSolrFieldName() + "^3 " +- CollectionSchema.text_t.getSolrFieldName() + "^1 " +- CollectionSchema.sku.getSolrFieldName() + "^0.5 " +- CollectionSchema.h1_txt.getSolrFieldName() + "^2");- params.set("pf",- CollectionSchema.title.getSolrFieldName() + "^5 " +- CollectionSchema.text_t.getSolrFieldName() + "^2");- //params.set("mm", "2<75%"); // using mm is too strict; in many cases we don't get any hits- final List<String> bqParts = new ArrayList<>();- if (boostTerms != null && !boostTerms.isEmpty()) {- for (String term : boostTerms) {- if (term == null || term.isEmpty()) continue;- bqParts.add(CollectionSchema.title.getSolrFieldName() + ":" + term + "^0.5");- bqParts.add(CollectionSchema.h1_txt.getSolrFieldName() + ":" + term + "^0.4");- bqParts.add(CollectionSchema.text_t.getSolrFieldName() + ":" + term + "^0.2");- }- }- bqParts.add("(" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(zip rar 7z tar gz bz2 xz tgz))^0.1");- params.set("bq", String.join(" ", bqParts));- params.setRows(count);- params.setStart(0);- params.setFacet(false);- params.clearSorts();- params.setFields(- CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName(),- CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.keywords.getSolrFieldName(), CollectionSchema.synonyms_sxt.getSolrFieldName(),- CollectionSchema.h1_txt.getSolrFieldName(), CollectionSchema.h2_txt.getSolrFieldName(), CollectionSchema.h3_txt.getSolrFieldName(),- CollectionSchema.h4_txt.getSolrFieldName(), CollectionSchema.h5_txt.getSolrFieldName(), CollectionSchema.h6_txt.getSolrFieldName()- );- params.setIncludeScore(true);- params.set("df", CollectionSchema.text_t.getSolrFieldName());-- // query the server- try {- final SolrDocumentList sdl = connector.getDocumentListByParams(params);- Iterator<SolrDocument> i = sdl.iterator();- while (i.hasNext()) {- try {- SolrDocument doc = i.next();- final JSONObject result = new JSONObject(true);- String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());- result.put("url", url == null ? "" : url.trim());- String title = getOneString(doc, CollectionSchema.title);- result.put("title", title == null ? "" : title.trim());- if (includeSnippet) {- String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());- result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));- }- results.put(result);- } catch (JSONException e) {- // skip this result- }- }- return results;- } catch (SolrException | IOException e) {- return results;- }- }-- public static String searchResultsAsMarkdown(String query, int count, boolean global) {- return searchResultsAsMarkdown(query, count, global, new LinkedHashSet<>());- }-- public static String searchResultsAsMarkdown(String query, int count, boolean global, final Set<String> boostTerms) {- final long searchStart = System.currentTimeMillis();- JSONArray searchResults = global ? searchResultsGlobal(query, count, true) : searchResults(query, count, true, boostTerms);- ConcurrentLog.info("RAGProxy", "searchResults=" + searchResults.length() + " global=" + global + " searchMs=" + (System.currentTimeMillis() - searchStart));- StringBuilder sb = new StringBuilder();-- // collect snippets- List<Snippet> results = new ArrayList<>();- for (int i = 0; i < searchResults.length(); i++) {- try {- JSONObject r = searchResults.getJSONObject(i);- String title = r.optString("title", "");- String url = r.optString("url", "");- String text = r.optString("text", "");- if (title.isEmpty()) title = url;- if (text.isEmpty()) text = title;- if (title.length() > 0 && text.length() > 0) {- Snippet snippet = new Snippet(query, text, url, title, 256); // we always compute a snippet because that gives us a hint if the query appears at all- if (snippet.getText().length() > 0) results.add(snippet);- }- } catch (JSONException e) {}- }-- // sort snippets again by score- results.sort(Comparator.comparingDouble(Snippet::getScore));-- int limit = results.size() / 2;- if (results.size() > 0 && limit == 0) limit = 1;- for (int i = 0; i < limit; i++) {- Snippet snippet = results.get(i);- sb.append("## ").append(snippet.getTitle()).append("\n");- sb.append(snippet.text).append("\n");- if (snippet.getURL().length() > 0) sb.append("Source: ").append(snippet.getURL()).append("\n");- sb.append("\n\n");- }-- ConcurrentLog.info("RAGProxy", "markdownChars=" + sb.length() + " snippetCount=" + results.size());- return sb.toString();- }-- public static JSONArray searchResultsGlobal(String query, int count, final boolean includeSnippet) {- final JSONArray results = new JSONArray();- if (query == null || query.length() == 0 || count == 0) return results;- final Switchboard sb = Switchboard.getSwitchboard();- final RankingProfile ranking = sb.getRanking();- final int timezoneOffset = 0;- final QueryModifier modifier = new QueryModifier(timezoneOffset);- String querystring = modifier.parse(query);- if (querystring.length() == 0) {- querystring = query == null ? "" : query.trim();- }- if (querystring.length() == 0) return results;- final QueryGoal qg = new QueryGoal(querystring);- final QueryParams theQuery = new QueryParams(- qg,- modifier,- 0,- "",- Classification.ContentDomain.TEXT,- "",- timezoneOffset,- new HashSet<Tagging.Metatag>(),- CacheStrategy.IFFRESH,- count,- 0,- ".*",- null,- null,- QueryParams.Searchdom.GLOBAL,- null,- true,- DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")),- MultiProtocolURL.TLD_any_zone_filter,- null,- false,- sb.index,- ranking,- ClientIdentification.yacyIntranetCrawlerAgent.userAgent(),- 0.0d,- 0.0d,- 0.0d,- sb.getConfigSet("search.navigation"));- final SearchEvent theSearch = SearchEventCache.getEvent(- theQuery,- sb.peers,- sb.tables,- (sb.isRobinsonMode()) ? sb.clusterhashes : null,- false,- sb.loader,- (int) sb.getConfigLong(- SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER,- sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_DEFAULT, 10)),- sb.getConfigLong(- SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,- sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)));- final long timeout = sb.getConfigLong(- SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,- sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000));- waitForFeedingAndResort(theSearch, timeout);- for (int i = 0; i < count; i++) {- URIMetadataNode node = theSearch.oneResult(i, timeout);- if (node == null) break;- try {- final JSONObject result = new JSONObject(true);- result.put("url", node.urlstring());- result.put("title", node.title());- if (includeSnippet) {- String text = node.snippet();- if (text == null || text.isEmpty()) {- TextSnippet snippet = node.textSnippet();- if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) {- text = snippet.getLineRaw();- }- }- if (text == null || text.isEmpty()) {- text = firstFieldString(node.getFieldValue(CollectionSchema.description_txt.getSolrFieldName()));- }- if (text == null || text.isEmpty()) {- text = firstFieldString(node.getFieldValue(CollectionSchema.text_t.getSolrFieldName()));- }- result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));- }- results.put(result);- } catch (JSONException e) {- // skip this result- }- }- return results;- }-- private static String limitSnippet(String text, int maxChars) {- if (text == null) return "";- if (maxChars <= 0 || text.length() <= maxChars) return text;- return text.substring(0, maxChars);- }-- private static String firstFieldString(Object value) {- if (value == null) return "";- if (value instanceof Collection) {- for (Object item : (Collection<?>) value) {- if (item != null) return item.toString();- }- return "";- }- return value.toString();- }-- private static void waitForFeedingAndResort(SearchEvent search, long timeoutMs) {- if (search == null || timeoutMs <= 0) return;- final long end = System.currentTimeMillis() + timeoutMs;- while (!search.isFeedingFinished() && System.currentTimeMillis() < end) {- try {- Thread.sleep(100);- } catch (InterruptedException e) {- Thread.currentThread().interrupt();- break;- }- }- search.resortCachedResults();- }--- public static class Snippet {-- private String text, url, title;- private double score;-- /**- * Find a snippet inside a given text that contains most of the searched words plus some context.- * @param query a string with a search query; query words are separated by space- * @param text the text where we want to find the snippets- * @param maxChunkLength the maximum length of a single chunk; however the snippet is three times as this.- * @return one string containing the snippet.- */- public Snippet(String query, String text, String url, String title, int maxChunkLength) {- this.url = url;- this.title = title;- this.score = 0.0;-- if (text == null || text.isEmpty() || maxChunkLength <= 0 || query == null) {- this.text = "";- return;- }-- // Step 1: Slice text and make copy with lowercase version to support tf*idf computation- List<String> chunks = slicer(text, maxChunkLength);- if (chunks.isEmpty()) {- this.text = "";- return;- }- List<String> chunksLowerCase = new ArrayList<>(chunks.size());- for (String chunk: chunks) chunksLowerCase.add(chunk.toLowerCase());-- // Step 2: Preprocess query- Set<String> queryWordSet = querySet(query);- if (queryWordSet.isEmpty()) {- this.text = "";- return;- }-- // Step 3: Compute IDF- // IDF uses a logarithm because the information gain of rare words grows non-linearly;- // the log dampens extreme ratios (N/df), stabilizes TF-IDF values, and matches the- // information-theoretic definition of word informativeness.- int totalChunks = chunksLowerCase.size();- Map<String, Double> idf = new HashMap<>();- for (String word: queryWordSet) {- int docFreq = 0;- for (String chunk: chunksLowerCase) {- if (chunk.contains(word)) docFreq++;- }- idf.put(word, Math.log((double) totalChunks / (docFreq + 1)) + 1);- }-- // Step 4: Score chunks- Map<Integer, Double> chunkScores = new HashMap<>();- for (int i = 0; i < chunksLowerCase.size(); i++) {- String chunk = chunksLowerCase.get(i);- double score = 0.0;- Map<String, Integer> tf = new HashMap<>(); // counts occurrence in query for each word in chunk-- // Extract words and clean- String[] wordsInChunk = chunk.split("\\s+");- for (String w : wordsInChunk) {- String cleanWord = w.replaceAll("[.,!?;:]", "");- if (cleanWord.length() > 0 && queryWordSet.contains(cleanWord)) {- tf.put(cleanWord, tf.getOrDefault(cleanWord, 0) + 1);- }- }-- // Sum TF-IDF- for (String word: queryWordSet) {- int tfValue = tf.getOrDefault(word, 0);- double tfIdf = (double) tfValue * idf.getOrDefault(word, 1.0);- score += tfIdf;- }- chunkScores.put(i, score);- }-- // Step 5: Find best chunk- int topChunkIndex = -1;- for (Map.Entry<Integer, Double> entry: chunkScores.entrySet()) {- if (entry.getValue() > this.score) {- this.score = entry.getValue();- topChunkIndex = entry.getKey();- }- }-- // if there is no best chunk, return an empty snippet- if (topChunkIndex < 0) {- this.text = "";- this.score = 0.0;- return;- }-- // Step 6: Get 3-slice snippet- List<String> snippetChunks = new ArrayList<>();- if (topChunkIndex > 0) {- snippetChunks.add(chunks.get(topChunkIndex - 1));- }- snippetChunks.add(chunks.get(topChunkIndex));- if (topChunkIndex < chunks.size() - 1) {- snippetChunks.add(chunks.get(topChunkIndex + 1));- }-- // Step 7: Join- this.text = String.join(" ", snippetChunks);- }-- public double getScore() {- return this.score;- }-- public String getText() {- return this.text;- }-- public String getURL() {- return this.url;- }-- public String getTitle() {- return this.title;- }- }-- /**- * Creates slices of a given text. We want slices of average same size,- * but we want to prevent that cuts are made within sentences.- * @param text the given text- * @param len the minimum length of the wanted slices; actual slices may be longer- * @return a list of text slices- */- public static List<String> slicer(String text, int len) {- List<String> result = new ArrayList<>();- if (text == null || len <= 0) return result;-- int start = 0;- while (start < text.length()) {- int end = Math.min(start + len, text.length());-- // Move end position further out until a a sentence end is found:- // look for sentence boundary: .!?, followed by whitespace char.- while (end < text.length()) {- char ch = text.charAt(end - 1);- if ((ch == '.' || ch == '?' || ch == '!') && Character.isWhitespace(text.charAt(end))) {- break;- }- end++;- }- result.add(text.substring(start, end));- start = end;- }-- return result;- }-- private static String getOneString(SolrDocument doc, CollectionSchema field) {- assert field.isMultiValued();- assert field.getType() == SolrType.string || field.getType() == SolrType.text_general;- Object r = doc.getFieldValue(field.getSolrFieldName());- if (r == null) return "";- if (r instanceof ArrayList) {- return (String) ((ArrayList<?>) r).get(0);- }- return r.toString();- }-- private String searchWordsForPrompt(LLM llm, String model, String prompt) {- final String question = prompt == null ? "" : prompt;- if (llm == null || model == null || model.isEmpty()) {- return null;- }- try {- LLM.Context context = new LLM.Context(LLM_SYSTEM_PREFIX_DEFAULT);- context.addPrompt(question);- Set<String> singlewords = new LinkedHashSet<>();- String[] a = LLM.stringsFromChat(llm.chat(model, context, LLM.listSchema, 200));- if (a == null || a.length == 0) return null;- // unfortunately this might not be a single word per line but several words; we collect them all.- for (String s: a) {- if (s == null) continue;- for (String t: s.split(" ")) {- if (!t.isEmpty()) singlewords.add(t.toLowerCase());- }- }- if (singlewords.isEmpty()) {- return null;- }- StringBuilder query = new StringBuilder();- for (String s: singlewords) query.append(s).append(' ');- String querys = query.toString().trim();- if (querys.length() == 0) return null;- return querys;- } catch (IOException | JSONException e) {- e.printStackTrace();- return null;- }- }-- private static Set<String> querySet(String query) {- Set<String> queryWordSet = Arrays.stream(query.trim().toLowerCase().split("\\s+"))- .map(String::toLowerCase)- .filter(word -> !word.isEmpty())- .collect(Collectors.toSet());- return queryWordSet;- }-- private static Set<String> intersectTokens(String originalPrompt, String computedQuery, int maxTerms) {- Set<String> promptTerms = querySet(originalPrompt == null ? "" : originalPrompt);- Set<String> queryTerms = querySet(computedQuery == null ? "" : computedQuery);- Set<String> intersection = new LinkedHashSet<>();- for (String term : promptTerms) {- if (!queryTerms.contains(term)) continue;- final String cleaned = cleanToken(term);- if (cleaned.isEmpty()) continue;- intersection.add(cleaned);- if (maxTerms > 0 && intersection.size() >= maxTerms) break;- }- return intersection;- }-- private static String cleanToken(String term) {- if (term == null) return "";- String cleaned = term.replaceAll("[^A-Za-z0-9]", "");- if (cleaned.length() < 2) return "";- return cleaned.toLowerCase();- }-- private static JSONObject responseLine(String payload) {- JSONObject j = new JSONObject(true);- try {- j.put("id", "log");- j.put("object", "chat.completion.chunk");- j.put("created", System.currentTimeMillis() / 1000);- j.put("model", "log");- j.put("system_fingerprint", "YaCy");- JSONArray choices = new JSONArray();- JSONObject choice = new JSONObject(true); // {"index":0,"delta":{"role":"assistant","content":"ߘ"- choice.put("index", 0);- JSONObject delta = new JSONObject(true);- delta.put("role", "assistant");- delta.put("content", payload);- choice.put("delta", delta);- choices.put(choice);- j.put("choices", choices);- // j.put("finish_reason", null); // this is problematic with the JSON library- } catch (JSONException e) {- }- return j;- }public static void pruneOldEntries(long now) {while (true) {