Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
added MCP server You can now use YaCy as a search tool within an agentic LLM framework, i.e. using MCP clients in N8N or OpenAI AgentBuilder. The tool address is /tools
Michael Peter Christenf41415a299ec
diff --git a/defaults/web.xml b/defaults/web.xmlindex 43a500cc1..d0e850709 100644--- a/defaults/web.xml+++ b/defaults/web.xml@@ -53,6 +53,11 @@<servlet><servlet-name>RAGProxyServlet</servlet-name><servlet-class>net.yacy.http.servlets.RAGProxyServlet</servlet-class>+ </servlet>++ <servlet>+ <servlet-name>MCPSearchServlet</servlet-name>+ <servlet-class>net.yacy.http.servlets.MCPSearchServlet</servlet-class></servlet><!-- servlet to provide searchresults via proxy -->@@ -92,6 +97,15 @@<servlet-name>RAGProxyServlet</servlet-name><url-pattern>/v1/chat/completions</url-pattern></servlet-mapping>++ <servlet-mapping>+ <servlet-name>MCPSearchServlet</servlet-name>+ <url-pattern>/tools</url-pattern>+ <url-pattern>/tools/list</url-pattern>+ <url-pattern>/tools/list_changed</url-pattern>+ <url-pattern>/tools/call</url-pattern>+ </servlet-mapping>+<!-- eof hardcoded mappings --><!-- additional (optional) mappings -->diff --git a/source/net/yacy/http/servlets/MCPSearchServlet.java b/source/net/yacy/http/servlets/MCPSearchServlet.javanew file mode 100644index 000000000..93a2db1b9--- /dev/null+++ b/source/net/yacy/http/servlets/MCPSearchServlet.java@@ -0,0 +1,323 @@+/**+ * MCPSearchServlet+ * Copyright 2025 by Michael Peter Christen+ * First released 11.10.2025 at https://yacy.net+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.http.servlets;++import java.io.BufferedReader;+import java.io.IOException;++import javax.servlet.ServletException;+import javax.servlet.ServletRequest;+import javax.servlet.ServletResponse;+import javax.servlet.http.HttpServlet;+import javax.servlet.http.HttpServletRequest;+import javax.servlet.http.HttpServletResponse;++import org.apache.solr.servlet.cache.Method;+import org.json.JSONArray;+import org.json.JSONException;+import org.json.JSONObject;+import org.json.JSONTokener;++/**+ * This servlet exposes a minimal Model Context Protocol (MCP) server that+ * offers a single tool, `search`, backed by the YaCy search index.+ * Clients can initialize a JSON-RPC session, list available tools and invoke+ * the web search tool to retrieve results from the embedded Solr instance.+ * The response payload follows the MCP conventions (JSON-RPC 2.0 with tool+ * results wrapped inside a `content` array).+ */+public class MCPSearchServlet extends HttpServlet {++ private static final long serialVersionUID = 433609077273989355L;++ private static final String JSONRPC_VERSION = "2.0";+ private static final String MCP_PROTOCOL_VERSION = "2024-11-05";+ private static final String TOOL_NAME = "search";+ private static final int DEFAULT_RESULT_COUNT = 10;+ private static final int MAX_RESULT_COUNT = 100;++ @Override+ public void service(final ServletRequest request, final ServletResponse response) throws ServletException, IOException {+ final HttpServletRequest hrequest = (HttpServletRequest) request;+ final HttpServletResponse hresponse = (HttpServletResponse) response;++ hresponse.setContentType("application/json;charset=utf-8");+ hresponse.setHeader("Access-Control-Allow-Origin", "*");+ hresponse.setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS, DELETE");+ hresponse.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization");++ final Method reqMethod = Method.getMethod(hrequest.getMethod());+ if (reqMethod == Method.OTHER) {+ hresponse.setStatus(HttpServletResponse.SC_OK);+ return;+ }++ final String body = readBody(request);+ Object parsed = null;+ if (body.length() == 0) {+ parsed = new JSONObject();+ } else try {+ final JSONTokener tokener = new JSONTokener(body);+ parsed = tokener.nextValue();+ } catch (JSONException e) {+ writeJsonResponse(hresponse, errorResponse(JSONObject.NULL, -32700, e.getMessage()));+ }++ try {+ if (parsed instanceof JSONObject) {+ if (((JSONObject) parsed).optString("method", "").length() == 0) {+ String uri = hrequest.getRequestURI();+ ((JSONObject) parsed).put("method", uri.substring(1));+ }+ final JSONObject responseObject = handleRequest((JSONObject) parsed);+ if (responseObject != null) {+ writeJsonResponse(hresponse, responseObject);+ } else {+ hresponse.setStatus(HttpServletResponse.SC_NO_CONTENT);+ }+ } else if (parsed instanceof JSONArray) {+ final JSONArray requestArray = (JSONArray) parsed;+ final JSONArray responseArray = new JSONArray();+ for (int i = 0; i < requestArray.length(); i++) {+ final Object entry = requestArray.get(i);+ if (entry instanceof JSONObject) {+ final JSONObject responseObject = handleRequest((JSONObject) entry);+ if (responseObject != null) {+ responseArray.put(responseObject);+ }+ }+ }+ if (responseArray.length() > 0) {+ writeJsonResponse(hresponse, responseArray);+ } else {+ hresponse.setStatus(HttpServletResponse.SC_NO_CONTENT);+ }+ } else {+ writeJsonResponse(hresponse, errorResponse(JSONObject.NULL, -32600, "Invalid JSON-RPC payload"));+ }+ } catch (JSONException e) {+ writeJsonResponse(hresponse, errorResponse(JSONObject.NULL, -32700, e.getMessage()));+ }+ }++ private JSONObject handleRequest(final JSONObject requestObject) {+ final Object id = requestObject.optString("id", "0");+ final String jsonrpc = requestObject.optString("jsonrpc", JSONRPC_VERSION);+ final String method = requestObject.optString("method", "");++ if (!JSONRPC_VERSION.equals(jsonrpc)) {+ return errorResponse(id, -32600, "Unsupported JSON-RPC version");+ }++ if (method == null || method.isEmpty()) {+ return errorResponse(id, -32600, "Missing method");+ }++ if (id == JSONObject.NULL || id == null) {+ // Notification: acknowledge silently+ if ("notifications/ping".equals(method)) {+ return null;+ }+ // No response for other notifications either+ return null;+ }++ switch (method) {+ case "initialize":+ return handleInitialize(id, requestObject.optJSONObject("params"));+ case "tools/list":+ return handleToolsList(id);+ case "tools/call":+ return handleToolsCall(id, requestObject.optJSONObject("params"));+ default:+ return errorResponse(id, -32601, "Unknown method: " + method);+ }+ }++ private JSONObject handleInitialize(final Object id, final JSONObject params) {+ final JSONObject result = new JSONObject(true);+ try {+ final JSONObject serverInfo = new JSONObject(true);+ serverInfo.put("name", "YaCy MCP Web Search");+ serverInfo.put("version", "1.0");++ final JSONObject capabilities = new JSONObject(true);+ final JSONObject toolsCapability = new JSONObject(true);+ toolsCapability.put("list", true);+ toolsCapability.put("call", true);+ capabilities.put("tools", toolsCapability);++ result.put("protocolVersion", MCP_PROTOCOL_VERSION);+ result.put("serverInfo", serverInfo);+ result.put("capabilities", capabilities);+ } catch (JSONException e) {+ return errorResponse(id, -32603, e.getMessage());+ }+ return successResponse(id, result);+ }++ private JSONObject handleToolsList(final Object id) {+ try {+ final JSONObject tool = new JSONObject(true);+ tool.put("name", TOOL_NAME);+ tool.put("description", "Search the YaCy index and return the most relevant web results.");++ final JSONObject inputProperties = new JSONObject(true);+ final JSONObject querySchema = new JSONObject(true);+ querySchema.put("type", "string");+ querySchema.put("description", "Search query to execute against the YaCy index.");+ inputProperties.put("query", querySchema);++ final JSONObject limitSchema = new JSONObject(true);+ limitSchema.put("type", "integer");+ limitSchema.put("minimum", 1);+ limitSchema.put("maximum", MAX_RESULT_COUNT);+ limitSchema.put("description", "Maximum number of results to return.");+ inputProperties.put("limit", limitSchema);++ final JSONObject includeSnippetSchema = new JSONObject(true);+ includeSnippetSchema.put("type", "boolean");+ includeSnippetSchema.put("description", "Include text snippets extracted from the indexed document content.");+ inputProperties.put("include_snippet", includeSnippetSchema);++ final JSONObject inputSchema = new JSONObject(true);+ inputSchema.put("type", "object");+ inputSchema.put("properties", inputProperties);+ final JSONArray required = new JSONArray();+ required.put("query");+ inputSchema.put("required", required);++ tool.put("inputSchema", inputSchema);++ final JSONObject outputSchema = new JSONObject(true);+ outputSchema.put("type", "object");+ final JSONObject outputProperties = new JSONObject(true);+ final JSONObject resultsSchema = new JSONObject(true);+ resultsSchema.put("type", "array");+ resultsSchema.put("description", "Ordered list of search results.");+ outputProperties.put("results", resultsSchema);+ outputSchema.put("properties", outputProperties);+ tool.put("outputSchema", outputSchema);++ final JSONArray tools = new JSONArray();+ tools.put(tool);++ final JSONObject result = new JSONObject(true);+ result.put("tools", tools);+ return successResponse(id, result);+ } catch (JSONException e) {+ return errorResponse(id, -32603, e.getMessage());+ }+ }++ private JSONObject handleToolsCall(final Object id, final JSONObject params) {+ if (params == null) {+ return errorResponse(id, -32602, "Missing params");+ }+ final String name = params.optString("name", "");+ if (!TOOL_NAME.equals(name)) {+ return errorResponse(id, -32601, "Unknown tool: " + name);+ }++ final JSONObject arguments = params.optJSONObject("arguments");+ if (arguments == null) {+ return errorResponse(id, -32602, "Missing tool arguments");+ }++ final String query = arguments.optString("query", "").trim();+ if (query.isEmpty()) {+ return errorResponse(id, -32602, "Tool argument 'query' must be a non-empty string");+ }++ int limit = arguments.optInt("limit", DEFAULT_RESULT_COUNT);+ if (limit <= 0) {+ limit = DEFAULT_RESULT_COUNT;+ }+ limit = Math.min(limit, MAX_RESULT_COUNT);+ final boolean includeSnippet = arguments.optBoolean("include_snippet", true);++ JSONArray results;+ results = RAGProxyServlet.searchResults(query, limit, includeSnippet);++ try {+ final JSONObject payload = new JSONObject(true);+ payload.put("results", results);++ final JSONObject contentItem = new JSONObject(true);+ contentItem.put("type", "json");+ contentItem.put("json", payload);++ final JSONArray content = new JSONArray();+ content.put(contentItem);++ final JSONObject result = new JSONObject(true);+ result.put("content", content);+ return successResponse(id, result);+ } catch (JSONException e) {+ return errorResponse(id, -32603, e.getMessage());+ }+ }++ private static String readBody(final ServletRequest request) throws IOException {+ final StringBuilder builder = new StringBuilder();+ try (BufferedReader reader = request.getReader()) {+ String line;+ while ((line = reader.readLine()) != null) {+ builder.append(line);+ }+ }+ return builder.toString();+ }++ private static JSONObject successResponse(final Object id, final JSONObject result) {+ final JSONObject response = new JSONObject(true);+ try {+ response.put("jsonrpc", JSONRPC_VERSION);+ response.put("id", id);+ response.put("result", result);+ } catch (JSONException e) {+ // As this method is only called with valid JSON objects we rethrow as unchecked+ throw new IllegalStateException("Failed to construct success response", e);+ }+ return response;+ }++ private static JSONObject errorResponse(final Object id, final int code, final String message) {+ final JSONObject response = new JSONObject(true);+ try {+ response.put("jsonrpc", JSONRPC_VERSION);+ response.put("id", id);+ final JSONObject error = new JSONObject(true);+ error.put("code", code);+ error.put("message", message);+ response.put("error", error);+ } catch (JSONException e) {+ throw new IllegalStateException("Failed to construct error response", e);+ }+ return response;+ }++ private static void writeJsonResponse(final HttpServletResponse response, final Object payload) throws IOException {+ final String serialized = payload instanceof JSONObject ? ((JSONObject) payload).toString()+ : payload instanceof JSONArray ? ((JSONArray) payload).toString() : payload.toString();+ response.getWriter().write(serialized);+ }+}diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.javaindex 638bd7372..bbc891c75 100644--- a/source/net/yacy/http/servlets/RAGProxyServlet.java+++ b/source/net/yacy/http/servlets/RAGProxyServlet.java@@ -28,8 +28,8 @@ import java.net.HttpURLConnection;import java.net.URI;import java.net.URISyntaxException;import java.net.URL;+import java.util.ArrayList;import java.util.Iterator;-import java.util.LinkedHashMap;import javax.servlet.ServletException;import javax.servlet.ServletOutputStream;@@ -49,6 +49,7 @@ import org.json.JSONException;import org.json.JSONObject;import net.yacy.ai.OpenAIClient;+import net.yacy.cora.federate.solr.SolrType;import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;import net.yacy.search.Switchboard;import net.yacy.search.schema.CollectionSchema;@@ -141,18 +142,16 @@ public class RAGProxyServlet extends HttpServlet {String query = this.searchWordsForPrompt(LLM_QUERY_MODEL, user);out.print(responseLine("Searching for '" + query + "'\n\n").toString() + "\n");out.flush();- LinkedHashMap<String, String> searchResults = searchResults(query, 4);- out.print(responseLine("Using the following sources for RAG:\n\n").toString() + "\n");- out.flush();- for (String s : searchResults.keySet()) {- out.print(responseLine("- `" + s + "`\n").toString() + "\n");- out.flush();- }+ JSONArray searchResults = searchResults(query, 4, true);out.print(responseLine("\n").toString());out.flush();system += LLM_SYSTEM_PREFIX;user += LLM_USER_PREFIX;- for (String s : searchResults.values()) user += s + "\n\n";+ for (int i = 0; i < searchResults.length(); i++) {+ JSONObject r = searchResults.getJSONObject(i);+ String snippet = r.optString("snippet", "");+ user += snippet + "\n\n";+ }systemObject.put("content", system);userObject.put("content", user);@@ -200,9 +199,9 @@ public class RAGProxyServlet extends HttpServlet {}}- public static LinkedHashMap<String, String> searchResults(String query, int count) {- LinkedHashMap<String, String> a = new LinkedHashMap<>();- if (query == null || query.length() == 0 || count == 0) return a;+ public static JSONArray searchResults(String query, int count, final boolean includeSnippet) {+ final JSONArray results = new JSONArray();+ if (query == null || query.length() == 0 || count == 0) return results;Switchboard sb = Switchboard.getSwitchboard();EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector();// construct query@@ -213,7 +212,7 @@ public class RAGProxyServlet extends HttpServlet {params.setFacet(false);params.clearSorts();params.setFields(CollectionSchema.sku.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName());- params.setIncludeScore(false);+ params.setIncludeScore(true);params.set("df", CollectionSchema.text_t.getSolrFieldName());// query the server@@ -221,15 +220,37 @@ public class RAGProxyServlet extends HttpServlet {final SolrDocumentList sdl = connector.getDocumentListByParams(params);Iterator<SolrDocument> i = sdl.iterator();while (i.hasNext()) {- SolrDocument doc = i.next();- String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());- String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());- a.put(url, text);+ try {+ SolrDocument doc = i.next();+ final JSONObject result = new JSONObject(true);+ String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());+ result.put("url", url);+ String title = getOneString(doc, CollectionSchema.title);+ result.put("title", title == null ? url : title);+ if (includeSnippet) {+ String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());+ result.put("snippet", text == null ? "" : text);+ }+ results.put(result);+ } catch (JSONException e) {+ // skip this result+ }}- return a;+ return results;} catch (SolrException | IOException e) {- return new LinkedHashMap<>();+ return results;+ }+ }++ private static String getOneString(SolrDocument doc, CollectionSchema field) {+ assert field.isMultiValued();+ assert field.getType() == SolrType.string || field.getType() == SolrType.text_general;+ Object r = doc.getFieldValue(field.getSolrFieldName());+ if (r == null) return "";+ if (r instanceof ArrayList) {+ return ((ArrayList<String>) r).get(0);}+ return r.toString();}private String searchWordsForPrompt(String model, String prompt) {