Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Started implementing optional https preference for protocol operations
luccioman17e004599d12
diff --git a/defaults/yacy.init b/defaults/yacy.initindex 31196f8ed..1d8c67449 100644--- a/defaults/yacy.init+++ b/defaults/yacy.init@@ -127,6 +127,10 @@ network.unit.domain.nocheck = false# that means it is not usable in YaCy p2p-configurations, only in private portal configurationsnetwork.unit.tenant.agent =+# Prefer https for in-protocol operations when available on remote peers+# A distinct general setting is available to control whether https sould be used for remote search queries : remotesearch.https.preferred+network.unit.protocol.https.preferred = false+# Update process properties# The update server location is given in the network.unit.definition,# but the settings for update processing and cycles are individual.diff --git a/htroot/MessageSend_p.java b/htroot/MessageSend_p.javaindex 52870ae78..57271bc00 100644--- a/htroot/MessageSend_p.java+++ b/htroot/MessageSend_p.java@@ -26,6 +26,7 @@//if the shell's current path is HTROOTimport java.io.IOException;+import java.net.MalformedURLException;import java.util.Date;import java.util.Map;@@ -34,6 +35,7 @@ import org.apache.http.entity.mime.content.ContentBody;import net.yacy.cora.date.GenericFormatter;import net.yacy.cora.document.encoding.ASCII;import net.yacy.cora.document.encoding.UTF8;+import net.yacy.cora.document.id.MultiProtocolURL;import net.yacy.cora.protocol.RequestHeader;import net.yacy.kelondro.util.FileUtils;import net.yacy.peers.Network;@@ -42,6 +44,7 @@ import net.yacy.peers.Protocol.Post;import net.yacy.peers.Seed;import net.yacy.peers.SeedDB;import net.yacy.search.Switchboard;+import net.yacy.search.SwitchboardConstants;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;import net.yacy.utils.crypt;@@ -77,8 +80,35 @@ public class MessageSend_p {// first ask if the other peer is online, and also what kind of document it acceptsSeed seed = sb.peers.getConnected(ASCII.getBytes(hash));if (seed != null) {- for (String ip : seed.getIPs()) {- final Map<String, String> result = Protocol.permissionMessage(seed.getPublicAddress(ip), hash);+ for (final String ip : seed.getIPs()) {+ Map<String, String> result = null;+ MultiProtocolURL targetBaseURL = null;+ final String targetBaseURLStr = seed.getPublicURL(ip,+ sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT));+ try {+ targetBaseURL = new MultiProtocolURL(targetBaseURLStr);+ result = Protocol.permissionMessage(targetBaseURL, seed, sb);+ } catch(final MalformedURLException e) {+ Network.log.warn("yacyClient.permissionMessage malformed target peer URL :" + targetBaseURLStr);+ } catch(final Exception e) {+ // most probably a network time-out exception+ Network.log.warn("yacyClient.permissionMessage error:" + e.getMessage());+ if(targetBaseURL.isHTTPS()) {+ try {+ /* Request made over https : retry using http on the same IP as a fallback */+ targetBaseURL = seed.getPublicMultiprotocolURL(ip, false);+ result = Protocol.permissionMessage(targetBaseURL, seed, sb);+ if(result != null) {+ /* Got a successfull result with http : mark now SSl as not available ont the target peer */+ seed.setFlagSSLAvailable(false);+ sb.peers.updateConnected(seed);+ }+ } catch (final IOException e2) {+ Network.log.warn("yacyClient.postMessage error:" + e2.getMessage());+ }+ }+ }//System.out.println("DEBUG: permission request result = " + result.toString());String peerName;Seed targetPeer = null;@@ -102,6 +132,7 @@ public class MessageSend_p {sb.peers.peerActions.interfaceDeparture(targetPeer, ip);}} else {+prop.put("mode_permission", "1");// write input form@@ -146,28 +177,56 @@ public class MessageSend_p {final String salt = crypt.randomSalt();// send request- final Map<String, ContentBody> parts = Protocol.basicRequestParts(Switchboard.getSwitchboard(), hash, salt);+ final Map<String, ContentBody> parts = Protocol.basicRequestParts(sb, hash, salt);parts.put("process", UTF8.StringBody("post"));parts.put("myseed", UTF8.StringBody(seedDB.mySeed().genSeedStr(salt)));parts.put("subject", UTF8.StringBody(subject));parts.put("message", UTF8.StringBody(mb));- Seed seed = seedDB.getConnected(ASCII.getBytes(hash));+ final Seed seed = seedDB.getConnected(ASCII.getBytes(hash));+ boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);Post post1 = null;- for (String ip: seed.getIPs()) {- try {- post1 = new Post(seed.getPublicAddress(ip), hash, "/yacy/message.html", parts, 20000);- } catch (IOException e) {- Network.log.warn("yacyClient.postMessage error:" + e.getMessage());- post1 = null;- }- if (post1 != null) break;- seedDB.peerActions.interfaceDeparture(seed, ip);+ for(final String ip : seed.getIPs()) {+ MultiProtocolURL targetBaseURL = null;+ try {+ targetBaseURL = seed.getPublicMultiprotocolURL(ip, preferHttps);+ post1 = new Post(targetBaseURL, seed.hash, "/yacy/message.html", parts, 20000);+ } catch(final MalformedURLException e) {+ Network.log.warn("yacyClient.postMessage malformed target peer URL when using ip " + ip);+ } catch (final IOException e) {+ Network.log.warn("yacyClient.postMessage error:" + e.getMessage());+ if(targetBaseURL.isHTTPS()) {+ try {+ /* Request made over https : retry using http on the same IP as a fallback */+ targetBaseURL = seed.getPublicMultiprotocolURL(ip, false);+ post1 = new Post(targetBaseURL, seed.hash, "/yacy/message.html", parts, 20000);+ if(post1 != null) {+ /* Got a successfull result with http : mark now SSl as not available ont the target peer */+ seed.setFlagSSLAvailable(false);+ }+ } catch (final IOException e2) {+ Network.log.warn("yacyClient.postMessage error:" + e2.getMessage());+ }+ }+ }++ if (post1 != null) {+ break;+ }+ seedDB.peerActions.interfaceDeparture(seed, ip);}- final Map<String, String> result1 = post1 == null ? null : FileUtils.table(post1.result);+ final Map<String, String> result1 = post1 == null ? null : FileUtils.table(post1.getResult());final Map<String, String> result = result1;- //message has been sent- prop.put("mode_status_response", result.get("response"));+ if(result != null) {+ // message has been sent+ prop.put("mode_status_response", result.get("response"));+ } else {+ prop.put("mode_status", "1");++ // "unresolved pattern", the remote peer is alive but had an exception+ prop.putXML("mode_status_message", message);+ }} catch (final NumberFormatException e) {prop.put("mode_status", "1");diff --git a/htroot/Network.java b/htroot/Network.javaindex ee5539dd8..11416b7e9 100644--- a/htroot/Network.java+++ b/htroot/Network.java@@ -27,6 +27,7 @@// javac -classpath .:../classes Network.java// if the shell's current path is HTROOT+import java.net.MalformedURLException;import java.util.HashMap;import java.util.HashSet;import java.util.Iterator;@@ -39,6 +40,7 @@ import java.util.regex.Matcher;import java.util.regex.Pattern;import java.util.regex.PatternSyntaxException;+import net.yacy.cora.document.id.MultiProtocolURL;import net.yacy.cora.protocol.ClientIdentification;import net.yacy.cora.protocol.Domains;import net.yacy.cora.protocol.HeaderFramework;@@ -223,29 +225,40 @@ public class Network {map.put(Seed.IP, challengeIP);map.put(Seed.PORT, challengePort);Seed peer = post.get("peerHash") == null ? null : new Seed(post.get("peerHash"), map);- String challengeAddress = peer.getPublicAddress(challengeIP);sb.updateMySeed();Seed mySeed = sb.peers.mySeed();- final Map<String, String> response = Protocol.hello(mySeed, sb.peers.peerActions, challengeAddress, peer.hash);-- if (response == null) {- Seed peerd = sb.peers.get(peer.hash);- if (peerd != null) peer = peerd;- sb.peers.peerActions.interfaceDeparture(peer, challengeIP);- prop.put("table_comment",1);- prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>");- } else {- String yourtype = response.get("yourtype");- String yourip = response.get("yourip");- peer = sb.peers.getConnected(peer.hash);- if (peer == null) {+ final String challengeURLStr = peer.getPublicURL(challengeIP, false);+ try {+ final MultiProtocolURL challengeURL = new MultiProtocolURL(challengeURLStr);+ final Map<String, String> response = Protocol.hello(mySeed, sb.peers.peerActions, challengeURL, peer.hash);++ if (response == null) {+ Seed peerd = sb.peers.get(peer.hash);+ if (peerd != null) peer = peerd;+ sb.peers.peerActions.interfaceDeparture(peer, challengeIP);prop.put("table_comment",1);- prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);+ prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from <a href=\"" + challengeURL + "\" target=\"_blank\">" + challengeURL + "</a>");} else {- prop.put("table_comment",2);- prop.put("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);- prop.putHTML("table_comment_details",peer.toString());+ String yourtype = response.get("yourtype");+ String yourip = response.get("yourip");+ peer = sb.peers.getConnected(peer.hash);+ if (peer == null) {+ prop.put("table_comment",1);+ prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from <a href=\"" + challengeURL + "\" target=\"_blank\">" + challengeURL + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);+ } else {+ prop.put("table_comment",2);+ prop.put("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at <a href=\"" + challengeURL + "\" target=\"_blank\">" + challengeURL + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);+ prop.putHTML("table_comment_details",peer.toString());+ }+ }+ } catch(final MalformedURLException e) {+ final Seed peerd = sb.peers.get(peer.hash);+ if (peerd != null) {+ peer = peerd;}+ sb.peers.peerActions.interfaceDeparture(peer, challengeIP);+ prop.put("table_comment",1);+ prop.put("table_comment_status", "publish: malformed URL for peer '" + peer.getName() + "/" + post.get("peerHash") + "' : " + challengeURLStr);}prop.putHTML("table_peerHash",post.get("peerHash"));diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.javaindex 5bed791be..a153204e9 100644--- a/htroot/yacy/hello.java+++ b/htroot/yacy/hello.java@@ -29,10 +29,12 @@import java.io.IOException;import java.net.InetAddress;+import java.net.MalformedURLException;import java.util.Iterator;import java.util.Set;import java.util.concurrent.ConcurrentMap;+import net.yacy.cora.document.id.MultiProtocolURL;import net.yacy.cora.protocol.Domains;import net.yacy.cora.protocol.HeaderFramework;import net.yacy.cora.protocol.RequestHeader;@@ -43,6 +45,7 @@ import net.yacy.peers.Seed;import net.yacy.peers.graphics.ProfilingGraph;import net.yacy.search.EventTracker;import net.yacy.search.Switchboard;+import net.yacy.search.SwitchboardConstants;import net.yacy.server.serverCore;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;@@ -153,11 +156,14 @@ public final class hello {}final int connectedBefore = sb.peers.sizeConnected();//ConcurrentLog.info("**hello-DEBUG**", "peer " + remoteSeed.getName() + " challenged us with IPs " + reportedips);+ final boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);+ final int totalTimeout = preferHttps ? 13000 : 6500;int callbackRemain = Math.min(5, reportedips.size());- long callbackStart = System.currentTimeMillis();- if (callbackRemain > 0 && reportedips.size() > 0) {+ final long callbackStart = System.currentTimeMillis();+ if (callbackRemain > 0 && reportedips.size() > 0) {for (String reportedip: reportedips) {- int partialtimeout = ((int) (callbackStart + 6500 - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented+ int partialtimeout = ((int) (callbackStart + totalTimeout - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implementedif (partialtimeout <= 0) break;//ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " is handled");if (Seed.isProperIP(reportedip)) {@@ -165,11 +171,24 @@ public final class hello {prop.put("yourip", reportedip);remoteSeed.setIP(reportedip);time = System.currentTimeMillis();- callback = Protocol.queryRWICount(remoteSeed.getPublicAddress(reportedip), remoteSeed.hash, partialtimeout);+ try {+ MultiProtocolURL remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, preferHttps);+ callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout);+ if (callback[0] < 0 && remoteBaseURL.isHTTPS()) {+ /* Failed using https : retry using http */+ remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, false);+ callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout);+ }+ } catch(final MalformedURLException e) {+ callback = new long[] {-1, -1};+ }//ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " returns callback " + (callback == null ? "NULL" : callback[0]));time_backping = System.currentTimeMillis() - time;backping_method = "reportedip=" + reportedip;- if (callback[0] >= 0) { success = true; break; }+ if (callback[0] >= 0) {+ success = true;+ break;+ }if (--callbackRemain <= 0) break; // no more tries left / restrict to a limited number of ips}}diff --git a/source/net/yacy/peers/Network.java b/source/net/yacy/peers/Network.javaindex 32a26fc59..53bff58c6 100644--- a/source/net/yacy/peers/Network.java+++ b/source/net/yacy/peers/Network.java@@ -54,6 +54,7 @@ import net.yacy.cora.document.encoding.ASCII;import net.yacy.cora.document.feed.RSSFeed;import net.yacy.cora.document.feed.RSSMessage;import net.yacy.cora.document.id.DigestURL;+import net.yacy.cora.document.id.MultiProtocolURL;import net.yacy.cora.protocol.Domains;import net.yacy.cora.util.ConcurrentLog;import net.yacy.peers.operation.yacySeedUploadFile;@@ -194,27 +195,51 @@ public class Networkprotected class publishThread extends Thread{- private Map<String, String> result;private final Seed seed;public publishThread(final ThreadGroup tg, final Seed seed) {super(tg, "PublishSeed_" + seed.getName());this.seed = seed;- this.result = null;}@Overridepublic final void run() {+ Map<String, String> result = null;try {- for (String ip: this.seed.getIPs()) {- this.result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed.getPublicAddress(ip), this.seed.hash);- if ( this.result == null ) {- // no or wrong response, delete that address- final String cause = "peer ping to peer resulted in error response (added < 0)";+ final boolean preferHttps = Network.this.sb.getConfigBool(+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);+ for (final String ip: this.seed.getIPs()) {+ try {+ MultiProtocolURL targetBaseURL = this.seed.getPublicMultiprotocolURL(ip, preferHttps);+ result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, targetBaseURL, this.seed.hash);+ if (result == null && targetBaseURL.isHTTPS()) {+ /* Failed with https : retry with http on the same address */+ targetBaseURL = this.seed.getPublicMultiprotocolURL(ip, false);+ result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions,+ targetBaseURL, this.seed.hash);+ if (result != null) {+ /* Got a result using http : mark SSL as unavailable on the peer */+ log.info("publish: SSL/TLS unavailable on " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '"+ + this.seed.getName() + "' : can be reached using http but not https on address "+ + ip);+ this.seed.setFlagSSLAvailable(false);+ Network.this.sb.peers.updateConnected(this.seed);+ }+ }+ if(result == null) {+ // no or wrong response, delete that address+ final String cause = "peer ping to peer resulted in error response (added < 0)";+ log.info("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getIPs() + ": " + cause);+ Network.this.sb.peers.peerActions.interfaceDeparture(this.seed, ip);+ continue;+ }+ } catch(final MalformedURLException e) {+ final String cause = "malformed peer URL";log.info("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getIPs() + ": " + cause);Network.this.sb.peers.peerActions.interfaceDeparture(this.seed, ip);- continue;- }+ continue;+ }// success! we have published our peer to a senior peer// update latest news from the other peerlog.info("publish: handshaked "+ this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs());diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.javaindex 94f9db8ed..8f5345681 100644--- a/source/net/yacy/peers/Protocol.java+++ b/source/net/yacy/peers/Protocol.java@@ -58,7 +58,6 @@ import java.util.LinkedHashMap;import java.util.LinkedHashSet;import java.util.List;import java.util.Map;-import java.util.Set;import java.util.TreeMap;import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.atomic.AtomicInteger;@@ -138,32 +137,37 @@ public final class Protocol {*/public static class Post {- public byte[] result; // contains the result from a successful post or null if no attempt was successful- public Set<String> unsuccessfulAddresses; // contains a set of addresses which had been tested for submission was without success- public String successfulAddress; // contains the address which had been successfully used or null if no success with any Address+ /** Contains the result from a successful post or null if no attempt was successful */+ private byte[] result;+ /**+ * @param targetBaseURL the base target URL+ * @param targetHash the hash of the target peer+ * @param path the path on the base URL+ * @param parts the body content+ * @param timeout the timeout in milliseconds+ * @param httpFallback when true, retry as http when a https request failed+ * @throws IOException+ */public Post(- final String targetAddress,- final String targetPeerHash,+ final MultiProtocolURL targetBaseURL,+ final String targetHash,final String path,final Map<String, ContentBody> parts,final int timeout) throws IOException {final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);httpClient.setTimout(timeout);- this.result = httpClient.POSTbytes(- new MultiProtocolURL("http://" + targetAddress + path),- Seed.b64Hash2hexHash(targetPeerHash) + ".yacyh",- parts,- false, true);- this.unsuccessfulAddresses = new HashSet<>();- if (this.result == null) {- this.unsuccessfulAddresses.add(targetAddress);- this.successfulAddress = null;- } else {- this.successfulAddress = targetAddress;- }+ MultiProtocolURL targetURL = new MultiProtocolURL(targetBaseURL, path);+ this.result = httpClient.POSTbytes(targetURL, Seed.b64Hash2hexHash(targetHash) + ".yacyh", parts, false,+ true);}-++ /**+ * @return the result from a successful post or null if no attempt was successful+ */+ public byte[] getResult() {+ return this.result;+ }}/**@@ -182,7 +186,7 @@ public final class Protocol {public static Map<String, String> hello(final Seed mySeed,final PeerActions peerActions,- final String targetAddress,+ final MultiProtocolURL targetBaseURL,final String targetHash) {Map<String, String> result = null;@@ -202,7 +206,7 @@ public final class Protocol {final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000);content =httpClient.POSTbytes(- new MultiProtocolURL("http://" + targetAddress + "/yacy/hello.html"),+ new MultiProtocolURL(targetBaseURL, "/yacy/hello.html"),Seed.b64Hash2hexHash(targetHash) + ".yacyh",parts,false, true);@@ -212,7 +216,7 @@ public final class Protocol {if ( Thread.currentThread().isInterrupted() ) {Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' interrupted.");return null;}- Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + targetAddress + "; exception: " + e.getMessage());+ Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + targetBaseURL + "; exception: " + e.getMessage());// try again (go into loop)result = null;}@@ -222,7 +226,7 @@ public final class Protocol {+ ((result == null) ? "result null" : ("result=" + result.toString())));return null;}- Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + targetAddress + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + responseTime + " milliseconds");+ Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + targetBaseURL + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + responseTime + " milliseconds");// check consistency with expectationSeed otherPeer = null;@@ -233,7 +237,7 @@ public final class Protocol {} else {try {// patch the remote peer address to avoid that remote peers spoof the network with wrong addresses- String host = Domains.stripToHostName(targetAddress);+ String host = targetBaseURL.getHost();InetAddress ie = Domains.dnsResolve(host);otherPeer = Seed.genRemoteSeed(seed, false, ie.getHostAddress());if ( !otherPeer.hash.equals(targetHash) ) {@@ -342,7 +346,7 @@ public final class Protocol {} else {try {if ( i == 1 ) {- String host = Domains.stripToHostName(targetAddress);+ String host = targetBaseURL.getHost();InetAddress ia = Domains.dnsResolve(host);if (ia == null) continue;host = ia.getHostAddress(); // the actual address of the target as we had been successful when contacting them is patched here@@ -364,23 +368,24 @@ public final class Protocol {return result;}- public static long[] queryRWICount(final String targetAddress, final String targetHash, int timeout) {+ public static long[] queryRWICount(final MultiProtocolURL targetBaseURL, final Seed target, int timeout) {// prepare requestfinal String salt = crypt.randomSalt();// send requesttry {- final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt);+ final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);parts.put("object", UTF8.StringBody("rwicount"));parts.put("env", UTF8.StringBody(""));- //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetAddress);- final Post post = new Post(targetAddress, targetHash, "/yacy/query.html", parts, timeout);- //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetAddress + (post.result == null ? "NULL" : (": length = " + post.result.length)));+ //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetBaseURL);+ final Post post = new Post(targetBaseURL, target.hash, "/yacy/query.html", parts, timeout);++ //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetBaseURL + (post.result == null ? "NULL" : (": length = " + post.result.length)));final Map<String, String> result = FileUtils.table(post.result);if (result == null || result.isEmpty()) return new long[] {-1, -1};- //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetAddress + " : result = " + result.toString());+ //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetBaseURL + " : result = " + result.toString());final String resp = result.get("response");- //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetAddress + " : response = " + resp);+ //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetBaseURL + " : response = " + resp);if (resp == null) return new long[] {-1, -1};String magic = result.get("magic");if (magic == null) magic = "0";@@ -390,7 +395,7 @@ public final class Protocol {return new long[] {-1, -1};}} catch (final Exception e ) {- //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetAddress + ": " + e.getMessage());+ //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetBaseURL + ": " + e.getMessage());if (Network.log.isFine()) Network.log.fine("yacyClient.queryRWICount error:" + e.getMessage());return new long[] {-1, -1};}@@ -518,7 +523,8 @@ public final class Protocol {Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");if(targetBaseURL.startsWith("https")) {/* First mark https unavailable on this peer before removing any interface */- target.setFlagSSLAvailable(false);+ target.setFlagSSLAvailable(false);+ event.peers.updateConnected(target);} else {event.peers.peerActions.interfaceDeparture(target, ip);}@@ -608,7 +614,8 @@ public final class Protocol {Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");if(targetBaseURL.startsWith("https")) {/* First mark https unavailable on this peer before removing any interface */- target.setFlagSSLAvailable(false);+ target.setFlagSSLAvailable(false);+ event.peers.updateConnected(target);} else {event.peers.peerActions.interfaceDeparture(target, ip);}@@ -1222,11 +1229,11 @@ public final class Protocol {if(targetBaseURL.startsWith("https")) {/* First mark https unavailable on this peer before removing anything else */target.setFlagSSLAvailable(false);+ event.peers.updateConnected(target);} else {target.setFlagSolrAvailable(false);}}- target.setFlagSolrAvailable(false || myseed);return -1;}} catch(InterruptedException e) {@@ -1477,26 +1484,34 @@ public final class Protocol {return true;}- public static Map<String, String> permissionMessage(final String targetAddress, final String targetHash) {- // ask for allowed message size and attachment size- // if this replies null, the peer does not answer-+ /**+ * Post a request asking for allowed message size and attachment size to the+ * target peer on the selected target ip. All parameters must not be null.+ *+ * @param targetBaseURL+ * the public base URL of the target peer on one of its reported IP+ * addresses in {@link Seed#getIPs()}+ * @param target+ * the target peer+ * @param sb+ * the switchboard instance+ * @return the result of the request+ * @throws IOException+ * when the peer doesn't answer on this IP or any other error+ * occurred+ */+ public static Map<String, String> permissionMessage(final MultiProtocolURL targetBaseURL, final Seed target,+ final Switchboard sb) throws IOException {// prepare requestfinal String salt = crypt.randomSalt();// send request- try {- final Map<String, ContentBody> parts =- basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt);- parts.put("process", UTF8.StringBody("permission"));- final Post post = new Post(targetAddress, targetAddress, "/yacy/message.html", parts, 6000);- final Map<String, String> result = FileUtils.table(post.result);- return result;- } catch (final Exception e ) {- // most probably a network time-out exception- Network.log.warn("yacyClient.permissionMessage error:" + e.getMessage());- return null;- }+ final Map<String, ContentBody> parts = basicRequestParts(sb, target.hash, salt);+ parts.put("process", UTF8.StringBody("permission"));+ final Post post = new Post(targetBaseURL, target.hash, "/yacy/message.html", parts, 6000);++ final Map<String, String> result = FileUtils.table(post.result);+ return result;}public static Map<String, String> crawlReceipt(@@ -1702,12 +1717,21 @@ public final class Protocol {final ReferenceContainerCache<WordReference> indexes,boolean gzipBody,final int timeout) {- for (String ip : targetSeed.getIPs()) {+ final boolean preferHttps = Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);+ for (final String ip : targetSeed.getIPs()) {if (ip == null) {Network.log.warn("no address for transferRWI");return null;}- final String address = targetSeed.getPublicAddress(ip);+ MultiProtocolURL targetBaseURL = null;+ try {+ targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, preferHttps);+ } catch(final MalformedURLException e) {+ Network.log.info("yacyClient.transferRWI malformed target URL : " + targetBaseURL);+ // disconnect unavailable peer ip+ Switchboard.getSwitchboard().peers.peerActions.interfaceDeparture(targetSeed, ip);+ continue;+ }// prepare post valuesfinal String salt = crypt.randomSalt();@@ -1746,12 +1770,25 @@ public final class Protocol {parts.put("entryc", UTF8.StringBody(Integer.toString(indexcount)));parts.put("indexes", UTF8.StringBody(entrypost.toString()));final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout);- final byte[] content =- httpClient.POSTbytes(- new MultiProtocolURL("http://" + address + "/yacy/transferRWI.html"),- targetSeed.getHexHash() + ".yacyh",- parts,- gzipBody, true);+ byte[] content = null;+ try {+ content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),+ targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);+ } catch(final IOException e) {+ if(targetBaseURL.isHTTPS()) {+ targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false);+ /* Failed with https : retry with http on the same address */+ content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),+ targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);+ if(content != null) {+ /* Success with http : mark SSL as unavailable on the target peer */+ Network.log.info("yacyClient.transferRWI SSL unavailable on address " + ip);+ Switchboard.getSwitchboard().peers.updateConnected(targetSeed);+ }+ } else {+ throw e;+ }+ }final Iterator<String> v = FileUtils.strings(content);// this should return a list of urlhashes that are unknown@@ -1761,7 +1798,7 @@ public final class Protocol {result.put(Seed.IP, ip); // add used ip to result for error handling (in case no "result" key was received)return result;} catch (final Exception e ) {- Network.log.info("yacyClient.transferRWI to " + address + " error: " + e.getMessage());+ Network.log.info("yacyClient.transferRWI to " + targetBaseURL + " error: " + e.getMessage());// disconnect unavailable peer ipSwitchboard.getSwitchboard().peers.peerActions.interfaceDeparture(targetSeed, ip);}@@ -1908,7 +1945,10 @@ public final class Protocol {final Map<String, ContentBody> parts =basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);parts.put("object", UTF8.StringBody("host"));- final Post post = new Post(target.getPublicAddress(target.getIP()), target.hash, "/yacy/idx.json", parts, 30000);+ final String remoteBaseURL = target.getPublicURL(target.getIP(),+ Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT));+ final Post post = new Post(new MultiProtocolURL(remoteBaseURL), target.hash, "/yacy/idx.json", parts, 30000);if ( post.result == null || post.result.length == 0 ) {Network.log.warn("yacyClient.loadIDXHosts error: empty result");return null;diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.javaindex 5ec1ce52c..562b0eb93 100644--- a/source/net/yacy/peers/Seed.java+++ b/source/net/yacy/peers/Seed.java@@ -68,6 +68,7 @@ import net.yacy.cora.date.AbstractFormatter;import net.yacy.cora.date.GenericFormatter;import net.yacy.cora.document.encoding.ASCII;import net.yacy.cora.document.encoding.UTF8;+import net.yacy.cora.document.id.MultiProtocolURL;import net.yacy.cora.federate.yacy.Distribution;import net.yacy.cora.order.Base64Order;import net.yacy.cora.order.Digest;@@ -791,7 +792,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>* @return an URL string for the given peer ip* @throws RuntimeException when the ip parameter is null*/- public final String getPublicURL(final String ip, final boolean preferHTTPS) {+ public final String getPublicURL(final String ip, final boolean preferHTTPS) throws RuntimeException {if (ip == null) {throw new RuntimeException("ip == NULL"); // that should not happen in Peer-to-Peer mode (but can in Intranet mode)}@@ -822,6 +823,27 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>}return sb.toString();}++ /**+ * Generate a public URL Multiprotocol instance using a given ip. This combines+ * the ip with the http(s) port and encloses the ip with square brackets if the+ * ip is of typeIPv6+ *+ * @param ip+ * a host name or ip address+ * @param preferHTTPS+ * when true and https is available on this Seed, use it as the+ * scheme part of the url+ * @return an MultiProtocolURL instance for the given peer ip+ * @throws RuntimeException+ * when the ip parameter is null+ * @throws MalformedURLException+ * when the ip and port could not make a well formed URL+ */+ public final MultiProtocolURL getPublicMultiprotocolURL(final String ip, final boolean preferHTTPS)+ throws RuntimeException, MalformedURLException {+ return new MultiProtocolURL(getPublicURL(ip, preferHTTPS));+ }/** @return the port number of this seed or <code>-1</code> if not present */public final int getPort() {diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.javaindex 88860ef77..71b91cab9 100644--- a/source/net/yacy/search/SwitchboardConstants.java+++ b/source/net/yacy/search/SwitchboardConstants.java@@ -520,6 +520,13 @@ public final class SwitchboardConstants {public static final String NETWORK_BOOTSTRAP_SEEDLIST_STUB = "network.unit.bootstrap.seedlist";public static final String NETWORK_SEARCHVERIFY = "network.unit.inspection.searchverify";++ /** Key of the setting controlling whether https should be preferred for in-protocol operations when available on remote peers.+ * A distinct general setting is available to control whether https sould be used for remote search queries : see {@link #REMOTESEARCH_HTTPS_PREFERRED} */+ public static final String NETWORK_PROTOCOL_HTTPS_PREFERRED = "network.unit.protocol.https.preferred";++ /** Default setting value controlling whether https should be preferred for in-protocol operations when available on remote peers */+ public static final boolean NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT = false;/*** appearance