Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Align RAG retrieval with web search and disable local post-ranking
Michael Peter Christen6f505d1f10ff
diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.javaindex df0e82242..8e9530e79 100644--- a/source/net/yacy/ai/LLM.java+++ b/source/net/yacy/ai/LLM.java@@ -272,6 +272,11 @@ public class LLM {data.put("messages", context);data.put("stop", new JSONArray(STOPTOKENS));data.put("stream", false);++ if (model.toLowerCase().contains("qwen3.5")) { // we don't think+ data.put("reasoning_effort", "none");+ data.put("enable_thinking", false);+ }if (schema != null) {System.out.println(schema.toString());diff --git a/source/net/yacy/ai/RAGAugmentor.java b/source/net/yacy/ai/RAGAugmentor.javaindex c9b7215a9..1b5723c3f 100644--- a/source/net/yacy/ai/RAGAugmentor.java+++ b/source/net/yacy/ai/RAGAugmentor.java@@ -189,11 +189,9 @@ public final class RAGAugmentor {String title = node.title();String text = null;if (includeSnippet) {+ TextSnippet snippet = node.textSnippet();+ if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) text = snippet.getLineRaw();if (text == null || text.isEmpty()) text = node.snippet();- if (text == null || text.isEmpty()) {- TextSnippet snippet = node.textSnippet();- if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) text = snippet.getLineRaw();- }if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.description_txt.getSolrFieldName()));if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.text_t.getSolrFieldName()));result.put("text", text == null ? "" : text.trim());@@ -264,6 +262,7 @@ public final class RAGAugmentor {theQuery.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));+ theQuery.setSnippetFetchFullText(true);theQuery.getQueryGoal().filterOut(Switchboard.blueList);return theQuery;}diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.javaindex e3bd008bf..4686f7cf5 100644--- a/source/net/yacy/search/query/QueryParams.java+++ b/source/net/yacy/search/query/QueryParams.java@@ -188,7 +188,9 @@ public final class QueryParams {public final int maxDistance;public final Bitfield constraint;public final boolean allofconstraint;- protected CacheStrategy snippetCacheStrategy;+ protected CacheStrategy snippetCacheStrategy;+ private boolean snippetFetchFullText;+ private boolean localSolrOnly;public final RankingProfile ranking;private final Segment indexSegment;public final String clienthost; // this is the client host that starts the query, not a site operator@@ -298,9 +300,11 @@ public final class QueryParams {this.zonecode = domainzone;this.constraint = constraint;this.allofconstraint = allofconstraint;- this.siteexcludes = siteexcludes != null && siteexcludes.isEmpty() ? null: siteexcludes;- this.snippetCacheStrategy = snippetCacheStrategy;- this.clienthost = host;+ this.siteexcludes = siteexcludes != null && siteexcludes.isEmpty() ? null: siteexcludes;+ this.snippetCacheStrategy = snippetCacheStrategy;+ this.snippetFetchFullText = false;+ this.localSolrOnly = false;+ this.clienthost = host;this.remotepeer = null;this.starttime = System.currentTimeMillis();this.maxtime = 10000;@@ -658,11 +662,11 @@ public final class QueryParams {if (bf.length() > 0) params.setParam("boost", bf); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29// set highlighting query attributes- if (this.contentdom == Classification.ContentDomain.TEXT || this.contentdom == Classification.ContentDomain.ALL) {- params.setHighlight(true);- params.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH);- //params.setHighlightRequireFieldMatch();- params.setHighlightSimplePost("</b>");+ if (!this.snippetFetchFullText && (this.contentdom == Classification.ContentDomain.TEXT || this.contentdom == Classification.ContentDomain.ALL)) {+ params.setHighlight(true);+ params.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH);+ //params.setHighlightRequireFieldMatch();+ params.setHighlightSimplePost("</b>");params.setHighlightSimplePre("<b>");params.setHighlightSnippets(5);for (final CollectionSchema field: SOLR_SNIPPET_FIELDS) {@@ -807,9 +811,24 @@ public final class QueryParams {} else {params.setFacet(false);}- params.setFields("*", "score"); // we need the score for post-ranking- return params;- }+ params.setFields(buildResultFieldList(this.snippetFetchFullText));+ return params;+ }++ private String[] buildResultFieldList(final boolean includeTextField) {+ final ArrayList<String> fields = new ArrayList<>();+ for (final CollectionSchema field : CollectionSchema.values()) {+ if (!this.solrSchema.contains(field) || !field.isStored()) {+ continue;+ }+ if (!includeTextField && field == CollectionSchema.text_t) {+ continue;+ }+ fields.add(field.getSolrFieldName());+ }+ fields.add("score");+ return fields.toArray(new String[fields.size()]);+ }long year = 1000L * 60L * 60L * 24L * 365L;@@ -942,7 +961,7 @@ public final class QueryParams {private volatile String idCacheAnon = null, idCache = null;final static private char asterisk = '*';- public String id(final boolean anonymized) {+ public String id(final boolean anonymized) {if (anonymized) {if (this.idCacheAnon != null) return this.idCacheAnon;} else {@@ -989,7 +1008,9 @@ public final class QueryParams {context.append(this.tld).append(asterisk);context.append(this.inlink).append(asterisk);context.append(this.lat).append(asterisk).append(this.lon).append(asterisk).append(this.radius).append(asterisk);- context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name());+ context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name()).append(asterisk);+ context.append(this.snippetFetchFullText).append(asterisk);+ context.append(this.localSolrOnly);// Note : this.maxSuggestions search parameter do not need to be part of this id, as it has no impact on results themselves@@ -1001,7 +1022,23 @@ public final class QueryParams {}return result;}- }+ }++ public boolean isSnippetFetchFullText() {+ return this.snippetFetchFullText;+ }++ public void setSnippetFetchFullText(final boolean snippetFetchFullText) {+ this.snippetFetchFullText = snippetFetchFullText;+ }++ public boolean isLocalSolrOnly() {+ return this.localSolrOnly;+ }++ public void setLocalSolrOnly(final boolean localSolrOnly) {+ this.localSolrOnly = localSolrOnly;+ }/*** Build a search query URL from the given parameters.diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.javaindex b7d7e94c2..9ad44a347 100644--- a/source/net/yacy/search/query/SearchEvent.java+++ b/source/net/yacy/search/query/SearchEvent.java@@ -227,6 +227,8 @@ public final class SearchEvent implements ScoreMapUpdatesListener {/** if this is true, then every entry in result List is polled immediately to prevent a re-ranking in the resultList. This is usefull if there is only one index source. */private final boolean pollImmediately;+ /** Disable post-ranking when operating in explicit local-search mode. */+ private final boolean disablePostRanking;public final boolean excludeintext_image;// the following values are filled during the search process as statistics for the search@@ -430,6 +432,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {this.feedersTerminated = new AtomicInteger(0);this.snippetFetchAlive = new AtomicInteger(0);this.addRunning = true;+ this.disablePostRanking = this.query.isLocal();this.receivedRemoteReferences = new AtomicInteger(0);this.order = new ReferenceOrder(this.query.ranking, this.query.targetlang);this.urlhashes = new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 100);@@ -496,7 +499,8 @@ public final class SearchEvent implements ScoreMapUpdatesListener {} else {this.primarySearchThreadsL = null;this.nodeSearchThreads = null;- this.pollImmediately = !query.getSegment().connectedRWI() || !Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, false);+ this.pollImmediately = !query.getSegment().connectedRWI()+ || !Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, false);if ( generateAbstracts ) {// we need the results nowtry {@@ -1893,9 +1897,11 @@ public final class SearchEvent implements ScoreMapUpdatesListener {this.query.getQueryGoal().getIncludeHashes(),CacheStrategy.CACHEONLY,false,- 180,+ snippetMaxLength(true),false);- final TextSnippet selectedSnippet = solrsnippet.getLineRaw().length() > yacysnippet.getLineRaw().length() ? solrsnippet : yacysnippet;+ final TextSnippet selectedSnippet = this.query.isSnippetFetchFullText() && node.getText() != null && !node.getText().isEmpty()+ ? new TextSnippet(node.url(), node.getText(), false, ResultClass.SOURCE_METADATA, "")+ : (solrsnippet.getLineRaw().length() > yacysnippet.getLineRaw().length() ? solrsnippet : yacysnippet);final URIMetadataNode re = node.makeResultEntry(this.query.getSegment(), this.peers, maximizeSnippet(node, selectedSnippet));addResult(re, localEntryElement.getWeight());success = true;@@ -1937,7 +1943,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {*/public void addResult(URIMetadataNode resultEntry, final long score) {if (resultEntry == null) return;- final long ranking = (score * 128) + postRanking(resultEntry, this.ref /*this.getTopicNavigator(MAX_TOPWORDS)*/);+ final long ranking = this.disablePostRanking ? score : (score * 128) + postRanking(resultEntry, this.ref /*this.getTopicNavigator(MAX_TOPWORDS)*/);// TODO: above was originally using (see below), but getTopicNavigator returns this.ref and possibliy alters this.ref on first call (this.ref.size < 2 -> this.ref.clear)// TODO: verify and straighten the use of addTopic, getTopic and getTopicNavigator and related score calculation// final long ranking = ((long) (score * 128.f)) + postRanking(resultEntry, this.getTopicNavigator(MAX_TOPWORDS));@@ -2011,6 +2017,14 @@ public final class SearchEvent implements ScoreMapUpdatesListener {public URIMetadataNode getSnippet(URIMetadataNode page, final CacheStrategy cacheStrategy) {if (page == null) return null;+ if (this.query.isSnippetFetchFullText()) {+ final String fullText = page.getText();+ if (fullText != null && !fullText.isEmpty()) {+ final TextSnippet snippet = new TextSnippet(page.url(), fullText, false, ResultClass.SOURCE_METADATA, "");+ return page.makeResultEntry(this.query.getSegment(), this.peers, snippet);+ }+ }+if (cacheStrategy == null) {final TextSnippet snippet = new TextSnippet(null,@@ -2019,7 +2033,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {this.snippetFetchWordHashes,null,((this.query.constraint != null) && (this.query.constraint.get(Tokenizer.flag_cat_indexof))),- SearchEvent.SNIPPET_MAX_LENGTH,+ snippetMaxLength(false),!this.query.isLocal());return page.makeResultEntry(this.query.getSegment(), this.peers, maximizeSnippet(page, snippet)); // result without snippet}@@ -2036,7 +2050,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {this.snippetFetchWordHashes,cacheStrategy,((this.query.constraint != null) && (this.query.constraint.get(Tokenizer.flag_cat_indexof))),- 180,+ snippetMaxLength(true),!this.query.isLocal());SearchEvent.log.info("text snippet load time for " + page.url().toNormalform(true) + ": " + (System.currentTimeMillis() - startTime) + " ms, " + (!snippet.getErrorCode().fail() ? "snippet found" : ("no snippet found (" + snippet.getError() + ")")));@@ -2082,10 +2096,13 @@ public final class SearchEvent implements ScoreMapUpdatesListener {return page.makeResultEntry(this.query.getSegment(), this.peers, null); // result without snippet}- private static TextSnippet maximizeSnippet(final URIMetadataNode page, final TextSnippet snippet) {+ private TextSnippet maximizeSnippet(final URIMetadataNode page, final TextSnippet snippet) {if (page == null || snippet == null || !snippet.exists()) {return snippet;}+ if (!this.query.isSnippetFetchFullText()) {+ return snippet;+ }String rawText = page.getText();if (rawText == null || rawText.isEmpty()) {final ArrayList<String> descriptions = page.getDescription();@@ -2099,6 +2116,13 @@ public final class SearchEvent implements ScoreMapUpdatesListener {return new TextSnippet(page.url(), rawText, snippet.getLineRaw(), snippet.isMarked(), snippet.getErrorCode(), snippet.getError());}+ private int snippetMaxLength(final boolean shortSnippetDefault) {+ if (this.query.isSnippetFetchFullText()) {+ return -1;+ }+ return shortSnippetDefault ? 180 : SearchEvent.SNIPPET_MAX_LENGTH;+ }+/*** This is the access point for the search interface to retrive ranked results.* for display.diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.javaindex f6c79bec9..383a8a587 100644--- a/source/net/yacy/search/snippet/TextSnippet.java+++ b/source/net/yacy/search/snippet/TextSnippet.java@@ -197,17 +197,18 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnipinit(url, rawLine, displayLine, isMarked, errorCode, errortext, beginTime);}- public TextSnippet(- final LoaderDispatcher loader,- final URIMetadataNode row,- final Set<String> queryTerms,- final HandleSet queryhashes,+ public TextSnippet(+ final LoaderDispatcher loader,+ final URIMetadataNode row,+ final Set<String> queryTerms,+ final HandleSet queryhashes,final CacheStrategy cacheStrategy,final boolean pre,final int snippetMaxLength,- final boolean reindexing) {- long beginTime = System.currentTimeMillis();- // heise = "0OQUNU3JSs05"+ final boolean reindexing) {+ long beginTime = System.currentTimeMillis();+ final boolean fullDocumentSnippet = snippetMaxLength < 0;+ // heise = "0OQUNU3JSs05"final DigestURL url = row.url();if (queryTerms.isEmpty()) {@@ -283,19 +284,24 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip// not found the snippetinit(url, null, null, false, ResultClass.SOURCE_METADATA, null, beginTime);return;- }-- if (sentences.iterator().hasNext()) {- try {- final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);- textline = tsr.getSnippet();- remainingTerms = tsr.getRemainingTerms();- } catch (final UnsupportedOperationException e) {- init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);- return;- }- }- }+ }++ if (sentences.iterator().hasNext()) {+ if (fullDocumentSnippet) {+ textline = collectAllSentences(sentences);+ remainingTerms.clear();+ } else {+ try {+ final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);+ textline = tsr.getSnippet();+ remainingTerms = tsr.getRemainingTerms();+ } catch (final UnsupportedOperationException e) {+ init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);+ return;+ }+ }+ }+ }if (remainingTerms.isEmpty()) {// we found the snippet or the query is fully included in the headline or url@@ -330,19 +336,23 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnipprevSentence = currentSentence;}}- }- if (textline == null) {- sentences.reset();- final StringBuilder s = new StringBuilder(snippetMaxLength);- for (final StringBuilder t: sentences) {- s.append(t).append(' ');- if (s.length() >= snippetMaxLength / 4 * 3) break;- }- if (s.length() > snippetMaxLength) { s.setLength(snippetMaxLength); s.trimToSize(); }- textline = s.toString();- }- }- }+ }+ if (textline == null) {+ sentences.reset();+ if (fullDocumentSnippet) {+ textline = collectAllSentences(sentences);+ } else {+ final StringBuilder s = new StringBuilder(snippetMaxLength);+ for (final StringBuilder t: sentences) {+ s.append(t).append(' ');+ if (s.length() >= snippetMaxLength / 4 * 3) break;+ }+ if (s.length() > snippetMaxLength) { s.setLength(snippetMaxLength); s.trimToSize(); }+ textline = s.toString();+ }+ }+ }+ }final String fallbackLine = textline.length() > 0 ? textline : this.line;init(url, fallbackLine, fallbackLine, false, ResultClass.SOURCE_METADATA, null, beginTime);return;@@ -397,28 +407,50 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnipreturn;}- try {- final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);- textline = tsr.getSnippet();- remainingTerms = tsr.getRemainingTerms();- } catch (final UnsupportedOperationException e) {- init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);- return;+ if (fullDocumentSnippet) {+ textline = collectAllSentences(sentences);+ remainingTerms.clear();+ } else {+ try {+ final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);+ textline = tsr.getSnippet();+ remainingTerms = tsr.getRemainingTerms();+ } catch (final UnsupportedOperationException e) {+ init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);+ return;+ }}- sentences = null;+ sentences = null;if (textline == null || !remainingTerms.isEmpty()) {init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "no matching snippet found", beginTime);return;}- if (textline.length() > snippetMaxLength) textline = textline.substring(0, snippetMaxLength);+ if (!fullDocumentSnippet && textline.length() > snippetMaxLength) textline = textline.substring(0, snippetMaxLength);// finally store this snippet in our own cacheif(wordhashes != null) {snippetsCache.put(wordhashes, urlHash, textline);}init(url, textline, textline, false, source, null, beginTime);- }+ }++ private static String collectAllSentences(final Iterable<StringBuilder> sentences) {+ if (sentences == null) {+ return null;+ }+ final StringBuilder all = new StringBuilder();+ for (final StringBuilder sentence : sentences) {+ if (sentence == null || sentence.length() == 0) {+ continue;+ }+ if (all.length() > 0) {+ all.append(' ');+ }+ all.append(sentence);+ }+ return all.toString();+ }/*** Init a snippet line for urlhash