Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Merge branch 'master' of https://github.com/yacy/yacy_search_server.git
luccioman6e1959f469de
diff --cc htroot/yacysearchitem.javaindex 8fa284c53,eaa1a9e30..d0ba27a06--- a/htroot/yacysearchitem.java+++ b/htroot/yacysearchitem.java@@@ -24,9 -24,11 +24,12 @@@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+import java.awt.Dimension;import java.io.File;+ import java.io.UnsupportedEncodingException;import java.net.MalformedURLException;+ import java.net.URLEncoder;+ import java.nio.charset.StandardCharsets;import java.util.Collection;import java.util.Date;import java.util.Iterator;@@@ -340,65 -335,70 +333,129 @@@ public class yacysearchitemreturn prop;}+ /**+ * Tries to retrieve favicon url from solr result document, or generates+ * default favicon URL (i.e. "http://host/favicon.ico") from resultURL and+ * port.+ *+ * @param result+ * solr document result. Must not be null.+ * @param preferredSize preferred icon size. If no one matches, most close icon is returned.+ * @return favicon URL or null when even default favicon URL can not be generated+ * @throws NullPointerException when one requested parameter is null+ */+ protected static DigestURL getFaviconURL(final URIMetadataNode result, Dimension preferredSize) {+ /*+ * We look preferably for a standard icon with preferred size, but+ * accept as a fallback other icons below 128x128 or with no known size+ */+ IconEntry faviconEntry = result.getFavicon(preferredSize);+ DigestURL faviconURL;+ if (faviconEntry == null) {+ try {+ String defaultFaviconURL = result.url().getProtocol() + "://" + result.url().getHost()+ + ((result.url().getPort() != -1) ? (":" + result.url().getPort()) : "") + "/favicon.ico";+ faviconURL = new DigestURL(defaultFaviconURL);+ } catch (final MalformedURLException e1) {+ ConcurrentLog.logException(e1);+ faviconURL = null;+ }+ } else {+ faviconURL = faviconEntry.getUrl();+ }++ return faviconURL;+ }+ /**+ * @param authenticated+ * true when current user is authenticated+ * @param faviconURL+ * url icon of web site+ * @return url to propose in search result or empty string when faviconURL+ * is null+ */+ private static String processFaviconURL(final boolean authenticated, DigestURL faviconURL) {+ final String iconUrlExt = MultiProtocolURL.getFileExtension(faviconURL.getFileName());+ /* Image format ouput for ViewFavicon servlet : default is png, except with gif and svg icons */+ final String viewFaviconExt = !iconUrlExt.isEmpty() && ImageViewer.isBrowserRendered(iconUrlExt) ? iconUrlExt : "png";++ /* Only use licence code for non authentified users. For authenticated users licence would never be released and would unnecessarily fill URLLicense.permissions. */+ StringBuilder contentFaviconURL = new StringBuilder();+ if (faviconURL != null) {+ contentFaviconURL.append("ViewFavicon.").append(viewFaviconExt).append("?maxwidth=16&maxheight=16&isStatic=true&quadratic");+ if (authenticated) {+ contentFaviconURL.append("&url=").append(faviconURL.toNormalform(true));+ } else {+ contentFaviconURL.append("&code=").append(URLLicense.aquireLicense(faviconURL));+ }+ }+ return contentFaviconURL.toString();+ }+++ /**+ * Add action links reserved to authorized users. All parameters must be non null.+ * @param sb the main Switchboard instance+ * @param prop properties map to feed+ * @param theSearch search event+ * @param resultUrlstring URL of the result item+ * @param resource resource scope ("local" or "global")+ * @param origQ origin query terms+ * @param urlhash URL hash of the result item+ */+ private static void addAuthorizedActions(final Switchboard sb, final serverObjects prop,+ final SearchEvent theSearch, final String resultUrlstring, final String resource, final String origQ,+ final String urlhash) {+ // check if url exists in bookmarks+ boolean bookmarkexists = sb.bookmarksDB.getBookmark(urlhash) != null;+ prop.put("content_authorized_bookmark", !bookmarkexists);+ // bookmark icon check for YMarks+ //prop.put("content_authorized_bookmark", sb.tables.bookmarks.hasBookmark("admin", urlhash) ? "0" : "1");++ /* Bookmark, delete and recommend action links share the same URL prefix */+ StringBuilder linkBuilder = new StringBuilder();+ String actionLinkPrefix = linkBuilder.append("yacysearch.html?query=").append(origQ.replace(' ', '+'))+ .append("&Enter=Search&count=").append(theSearch.query.itemsPerPage()).append("&offset=")+ .append((theSearch.query.neededResults() - theSearch.query.itemsPerPage())).append("&resource=")+ .append(resource).append("&time=3").toString();+ linkBuilder.setLength(0);++ String encodedURLString;+ try {+ encodedURLString = URLEncoder.encode(crypt.simpleEncode(resultUrlstring), StandardCharsets.UTF_8.name());+ } catch (UnsupportedEncodingException e1) {+ ConcurrentLog.warn("YACY_SEARCH_ITEM", "UTF-8 encoding is not supported!");+ encodedURLString = crypt.simpleEncode(resultUrlstring);+ }+ String bookmarkLink = linkBuilder.append(actionLinkPrefix).append("&bookmarkref=").append(urlhash)+ .append("&bookmarkurl=").append(encodedURLString).append("&urlmaskfilter=.*")+ .toString();+ linkBuilder.setLength(0);++ /* Delete and recommend action links share the same URL suffix */+ String encodedRanking;+ try {+ encodedRanking = URLEncoder.encode(crypt.simpleEncode(theSearch.query.ranking.toExternalString()), StandardCharsets.UTF_8.name());+ } catch (UnsupportedEncodingException e1) {+ ConcurrentLog.warn("YACY_SEARCH_ITEM", "UTF-8 encoding is not supported!");+ encodedRanking = crypt.simpleEncode(resultUrlstring);+ }+ String actionLinkSuffix = linkBuilder.append(urlhash)+ .append("&urlmaskfilter=.*").append("&order=").append(encodedRanking).toString();+ linkBuilder.setLength(0);++ String deleteLink = linkBuilder.append(actionLinkPrefix).append("&deleteref=").append(actionLinkSuffix).toString();+ linkBuilder.setLength(0);+ String recommendLink = linkBuilder.append(actionLinkPrefix).append("&recommendref=").append(actionLinkSuffix).toString();+ linkBuilder.setLength(0);++ prop.put("content_authorized_bookmark_bookmarklink", bookmarkLink);+ prop.put("content_authorized_recommend_deletelink", deleteLink);+ prop.put("content_authorized_recommend_recommendlink", recommendLink);++ prop.put("content_authorized_recommend", (sb.peers.newsPool.getSpecific(NewsPool.OUTGOING_DB, NewsPool.CATEGORY_SURFTIPP_ADD, "url", resultUrlstring) == null) ? "1" : "0");+ prop.put("content_authorized_urlhash", urlhash);+ }/**diff --cc source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.javaindex 621c6dcca,b087e3d5a..cac8f685d--- a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java@@@ -30,6 -30,15 +30,13 @@@ import java.util.LinkedHashSetimport java.util.List;import java.util.Map;-import net.yacy.cora.document.analysis.Classification;+ import net.yacy.cora.document.id.MultiProtocolURL;+ import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter.ResHead;+ import net.yacy.cora.protocol.HeaderFramework;+ import net.yacy.cora.util.ConcurrentLog;+ import net.yacy.cora.util.JSONObject;-import net.yacy.data.URLLicense;+ import net.yacy.search.schema.CollectionSchema;+import org.apache.lucene.document.Document;import org.apache.lucene.index.IndexableField;import org.apache.solr.common.util.NamedList;@@@ -42,14 -51,7 +49,6 @@@ import org.apache.solr.search.DocIteratimport org.apache.solr.search.DocList;import org.apache.solr.search.SolrIndexSearcher;- import net.yacy.cora.document.id.MultiProtocolURL;- import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter.ResHead;- import net.yacy.cora.protocol.HeaderFramework;- import net.yacy.cora.util.ConcurrentLog;- import net.yacy.search.schema.CollectionSchema;- import net.yacy.server.serverObjects;---/*** write the opensearch result in YaCys special way to include as much as in opensearch is included.* This will also include YaCy facets.diff --cc source/net/yacy/search/schema/CollectionConfiguration.javaindex bd20c4ce2,fa281c946..9e954dc73--- a/source/net/yacy/search/schema/CollectionConfiguration.java+++ b/source/net/yacy/search/schema/CollectionConfiguration.java@@@ -544,14 -522,11 +525,14 @@@ public class CollectionConfiguration exSubgraph subgraph = new Subgraph(inboundLinks.size(), outboundLinks.size());int c = 0;- final Object parser = document.getParserObject();+ final Object scraper = document.getScraperObject();boolean containsCanonical = false;DigestURL canonical = null;++ processIcons(doc, allAttr, inboundLinks, outboundLinks, document.getIcons().values());+- if (parser instanceof ContentScraper) {- final ContentScraper html = (ContentScraper) parser;+ if (scraper instanceof ContentScraper) {+ final ContentScraper html = (ContentScraper) scraper;List<ImageEntry> images = html.getImages();// header tags