Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

Started implementing optional https preference for protocol operations

luccioman17e004599d12
diff --git a/defaults/yacy.init b/defaults/yacy.init
index 31196f8ed..1d8c67449 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -127,6 +127,10 @@ network.unit.domain.nocheck = false
# that means it is not usable in YaCy p2p-configurations, only in private portal configurations
network.unit.tenant.agent =
+# Prefer https for in-protocol operations when available on remote peers
+# A distinct general setting is available to control whether https sould be used for remote search queries : remotesearch.https.preferred
+network.unit.protocol.https.preferred = false
+
# Update process properties
# The update server location is given in the network.unit.definition,
# but the settings for update processing and cycles are individual.
diff --git a/htroot/MessageSend_p.java b/htroot/MessageSend_p.java
index 52870ae78..57271bc00 100644
--- a/htroot/MessageSend_p.java
+++ b/htroot/MessageSend_p.java
@@ -26,6 +26,7 @@
//if the shell's current path is HTROOT
import java.io.IOException;
+import java.net.MalformedURLException;
import java.util.Date;
import java.util.Map;
@@ -34,6 +35,7 @@ import org.apache.http.entity.mime.content.ContentBody;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
+import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.Network;
@@ -42,6 +44,7 @@ import net.yacy.peers.Protocol.Post;
import net.yacy.peers.Seed;
import net.yacy.peers.SeedDB;
import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.utils.crypt;
@@ -77,8 +80,35 @@ public class MessageSend_p {
// first ask if the other peer is online, and also what kind of document it accepts
Seed seed = sb.peers.getConnected(ASCII.getBytes(hash));
if (seed != null) {
- for (String ip : seed.getIPs()) {
- final Map<String, String> result = Protocol.permissionMessage(seed.getPublicAddress(ip), hash);
+ for (final String ip : seed.getIPs()) {
+ Map<String, String> result = null;
+ MultiProtocolURL targetBaseURL = null;
+ final String targetBaseURLStr = seed.getPublicURL(ip,
+ sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT));
+ try {
+ targetBaseURL = new MultiProtocolURL(targetBaseURLStr);
+ result = Protocol.permissionMessage(targetBaseURL, seed, sb);
+ } catch(final MalformedURLException e) {
+ Network.log.warn("yacyClient.permissionMessage malformed target peer URL :" + targetBaseURLStr);
+ } catch(final Exception e) {
+ // most probably a network time-out exception
+ Network.log.warn("yacyClient.permissionMessage error:" + e.getMessage());
+ if(targetBaseURL.isHTTPS()) {
+ try {
+ /* Request made over https : retry using http on the same IP as a fallback */
+ targetBaseURL = seed.getPublicMultiprotocolURL(ip, false);
+ result = Protocol.permissionMessage(targetBaseURL, seed, sb);
+ if(result != null) {
+ /* Got a successfull result with http : mark now SSl as not available ont the target peer */
+ seed.setFlagSSLAvailable(false);
+ sb.peers.updateConnected(seed);
+ }
+ } catch (final IOException e2) {
+ Network.log.warn("yacyClient.postMessage error:" + e2.getMessage());
+ }
+ }
+ }
//System.out.println("DEBUG: permission request result = " + result.toString());
String peerName;
Seed targetPeer = null;
@@ -102,6 +132,7 @@ public class MessageSend_p {
sb.peers.peerActions.interfaceDeparture(targetPeer, ip);
}
} else {
+
prop.put("mode_permission", "1");
// write input form
@@ -146,28 +177,56 @@ public class MessageSend_p {
final String salt = crypt.randomSalt();
// send request
- final Map<String, ContentBody> parts = Protocol.basicRequestParts(Switchboard.getSwitchboard(), hash, salt);
+ final Map<String, ContentBody> parts = Protocol.basicRequestParts(sb, hash, salt);
parts.put("process", UTF8.StringBody("post"));
parts.put("myseed", UTF8.StringBody(seedDB.mySeed().genSeedStr(salt)));
parts.put("subject", UTF8.StringBody(subject));
parts.put("message", UTF8.StringBody(mb));
- Seed seed = seedDB.getConnected(ASCII.getBytes(hash));
+ final Seed seed = seedDB.getConnected(ASCII.getBytes(hash));
+ boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);
Post post1 = null;
- for (String ip: seed.getIPs()) {
- try {
- post1 = new Post(seed.getPublicAddress(ip), hash, "/yacy/message.html", parts, 20000);
- } catch (IOException e) {
- Network.log.warn("yacyClient.postMessage error:" + e.getMessage());
- post1 = null;
- }
- if (post1 != null) break;
- seedDB.peerActions.interfaceDeparture(seed, ip);
+ for(final String ip : seed.getIPs()) {
+ MultiProtocolURL targetBaseURL = null;
+ try {
+ targetBaseURL = seed.getPublicMultiprotocolURL(ip, preferHttps);
+ post1 = new Post(targetBaseURL, seed.hash, "/yacy/message.html", parts, 20000);
+ } catch(final MalformedURLException e) {
+ Network.log.warn("yacyClient.postMessage malformed target peer URL when using ip " + ip);
+ } catch (final IOException e) {
+ Network.log.warn("yacyClient.postMessage error:" + e.getMessage());
+ if(targetBaseURL.isHTTPS()) {
+ try {
+ /* Request made over https : retry using http on the same IP as a fallback */
+ targetBaseURL = seed.getPublicMultiprotocolURL(ip, false);
+ post1 = new Post(targetBaseURL, seed.hash, "/yacy/message.html", parts, 20000);
+ if(post1 != null) {
+ /* Got a successfull result with http : mark now SSl as not available ont the target peer */
+ seed.setFlagSSLAvailable(false);
+ }
+ } catch (final IOException e2) {
+ Network.log.warn("yacyClient.postMessage error:" + e2.getMessage());
+ }
+ }
+ }
+
+ if (post1 != null) {
+ break;
+ }
+ seedDB.peerActions.interfaceDeparture(seed, ip);
}
- final Map<String, String> result1 = post1 == null ? null : FileUtils.table(post1.result);
+ final Map<String, String> result1 = post1 == null ? null : FileUtils.table(post1.getResult());
final Map<String, String> result = result1;
- //message has been sent
- prop.put("mode_status_response", result.get("response"));
+ if(result != null) {
+ // message has been sent
+ prop.put("mode_status_response", result.get("response"));
+ } else {
+ prop.put("mode_status", "1");
+
+ // "unresolved pattern", the remote peer is alive but had an exception
+ prop.putXML("mode_status_message", message);
+ }
} catch (final NumberFormatException e) {
prop.put("mode_status", "1");
diff --git a/htroot/Network.java b/htroot/Network.java
index ee5539dd8..11416b7e9 100644
--- a/htroot/Network.java
+++ b/htroot/Network.java
@@ -27,6 +27,7 @@
// javac -classpath .:../classes Network.java
// if the shell's current path is HTROOT
+import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -39,6 +40,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
+import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
@@ -223,29 +225,40 @@ public class Network {
map.put(Seed.IP, challengeIP);
map.put(Seed.PORT, challengePort);
Seed peer = post.get("peerHash") == null ? null : new Seed(post.get("peerHash"), map);
- String challengeAddress = peer.getPublicAddress(challengeIP);
sb.updateMySeed();
Seed mySeed = sb.peers.mySeed();
- final Map<String, String> response = Protocol.hello(mySeed, sb.peers.peerActions, challengeAddress, peer.hash);
-
- if (response == null) {
- Seed peerd = sb.peers.get(peer.hash);
- if (peerd != null) peer = peerd;
- sb.peers.peerActions.interfaceDeparture(peer, challengeIP);
- prop.put("table_comment",1);
- prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>");
- } else {
- String yourtype = response.get("yourtype");
- String yourip = response.get("yourip");
- peer = sb.peers.getConnected(peer.hash);
- if (peer == null) {
+ final String challengeURLStr = peer.getPublicURL(challengeIP, false);
+ try {
+ final MultiProtocolURL challengeURL = new MultiProtocolURL(challengeURLStr);
+ final Map<String, String> response = Protocol.hello(mySeed, sb.peers.peerActions, challengeURL, peer.hash);
+
+ if (response == null) {
+ Seed peerd = sb.peers.get(peer.hash);
+ if (peerd != null) peer = peerd;
+ sb.peers.peerActions.interfaceDeparture(peer, challengeIP);
prop.put("table_comment",1);
- prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);
+ prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from <a href=\"" + challengeURL + "\" target=\"_blank\">" + challengeURL + "</a>");
} else {
- prop.put("table_comment",2);
- prop.put("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at <a href=\"http://" + challengeAddress + "\" target=\"_blank\">" + challengeAddress + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);
- prop.putHTML("table_comment_details",peer.toString());
+ String yourtype = response.get("yourtype");
+ String yourip = response.get("yourip");
+ peer = sb.peers.getConnected(peer.hash);
+ if (peer == null) {
+ prop.put("table_comment",1);
+ prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from <a href=\"" + challengeURL + "\" target=\"_blank\">" + challengeURL + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);
+ } else {
+ prop.put("table_comment",2);
+ prop.put("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at <a href=\"" + challengeURL + "\" target=\"_blank\">" + challengeURL + "</a>, yourtype = " + yourtype + ", yourip = " + yourip);
+ prop.putHTML("table_comment_details",peer.toString());
+ }
+ }
+ } catch(final MalformedURLException e) {
+ final Seed peerd = sb.peers.get(peer.hash);
+ if (peerd != null) {
+ peer = peerd;
}
+ sb.peers.peerActions.interfaceDeparture(peer, challengeIP);
+ prop.put("table_comment",1);
+ prop.put("table_comment_status", "publish: malformed URL for peer '" + peer.getName() + "/" + post.get("peerHash") + "' : " + challengeURLStr);
}
prop.putHTML("table_peerHash",post.get("peerHash"));
diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java
index 5bed791be..a153204e9 100644
--- a/htroot/yacy/hello.java
+++ b/htroot/yacy/hello.java
@@ -29,10 +29,12 @@
import java.io.IOException;
import java.net.InetAddress;
+import java.net.MalformedURLException;
import java.util.Iterator;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
+import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
@@ -43,6 +45,7 @@ import net.yacy.peers.Seed;
import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverCore;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@@ -153,11 +156,14 @@ public final class hello {
}
final int connectedBefore = sb.peers.sizeConnected();
//ConcurrentLog.info("**hello-DEBUG**", "peer " + remoteSeed.getName() + " challenged us with IPs " + reportedips);
+ final boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);
+ final int totalTimeout = preferHttps ? 13000 : 6500;
int callbackRemain = Math.min(5, reportedips.size());
- long callbackStart = System.currentTimeMillis();
- if (callbackRemain > 0 && reportedips.size() > 0) {
+ final long callbackStart = System.currentTimeMillis();
+ if (callbackRemain > 0 && reportedips.size() > 0) {
for (String reportedip: reportedips) {
- int partialtimeout = ((int) (callbackStart + 6500 - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented
+ int partialtimeout = ((int) (callbackStart + totalTimeout - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented
if (partialtimeout <= 0) break;
//ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " is handled");
if (Seed.isProperIP(reportedip)) {
@@ -165,11 +171,24 @@ public final class hello {
prop.put("yourip", reportedip);
remoteSeed.setIP(reportedip);
time = System.currentTimeMillis();
- callback = Protocol.queryRWICount(remoteSeed.getPublicAddress(reportedip), remoteSeed.hash, partialtimeout);
+ try {
+ MultiProtocolURL remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, preferHttps);
+ callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout);
+ if (callback[0] < 0 && remoteBaseURL.isHTTPS()) {
+ /* Failed using https : retry using http */
+ remoteBaseURL = remoteSeed.getPublicMultiprotocolURL(reportedip, false);
+ callback = Protocol.queryRWICount(remoteBaseURL, remoteSeed, partialtimeout);
+ }
+ } catch(final MalformedURLException e) {
+ callback = new long[] {-1, -1};
+ }
//ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " returns callback " + (callback == null ? "NULL" : callback[0]));
time_backping = System.currentTimeMillis() - time;
backping_method = "reportedip=" + reportedip;
- if (callback[0] >= 0) { success = true; break; }
+ if (callback[0] >= 0) {
+ success = true;
+ break;
+ }
if (--callbackRemain <= 0) break; // no more tries left / restrict to a limited number of ips
}
}
diff --git a/source/net/yacy/peers/Network.java b/source/net/yacy/peers/Network.java
index 32a26fc59..53bff58c6 100644
--- a/source/net/yacy/peers/Network.java
+++ b/source/net/yacy/peers/Network.java
@@ -54,6 +54,7 @@ import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.feed.RSSFeed;
import net.yacy.cora.document.feed.RSSMessage;
import net.yacy.cora.document.id.DigestURL;
+import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.peers.operation.yacySeedUploadFile;
@@ -194,27 +195,51 @@ public class Network
protected class publishThread extends Thread
{
- private Map<String, String> result;
private final Seed seed;
public publishThread(final ThreadGroup tg, final Seed seed) {
super(tg, "PublishSeed_" + seed.getName());
this.seed = seed;
- this.result = null;
}
@Override
public final void run() {
+ Map<String, String> result = null;
try {
- for (String ip: this.seed.getIPs()) {
- this.result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed.getPublicAddress(ip), this.seed.hash);
- if ( this.result == null ) {
- // no or wrong response, delete that address
- final String cause = "peer ping to peer resulted in error response (added < 0)";
+ final boolean preferHttps = Network.this.sb.getConfigBool(
+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);
+ for (final String ip: this.seed.getIPs()) {
+ try {
+ MultiProtocolURL targetBaseURL = this.seed.getPublicMultiprotocolURL(ip, preferHttps);
+ result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, targetBaseURL, this.seed.hash);
+ if (result == null && targetBaseURL.isHTTPS()) {
+ /* Failed with https : retry with http on the same address */
+ targetBaseURL = this.seed.getPublicMultiprotocolURL(ip, false);
+ result = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions,
+ targetBaseURL, this.seed.hash);
+ if (result != null) {
+ /* Got a result using http : mark SSL as unavailable on the peer */
+ log.info("publish: SSL/TLS unavailable on " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '"
+ + this.seed.getName() + "' : can be reached using http but not https on address "
+ + ip);
+ this.seed.setFlagSSLAvailable(false);
+ Network.this.sb.peers.updateConnected(this.seed);
+ }
+ }
+ if(result == null) {
+ // no or wrong response, delete that address
+ final String cause = "peer ping to peer resulted in error response (added < 0)";
+ log.info("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getIPs() + ": " + cause);
+ Network.this.sb.peers.peerActions.interfaceDeparture(this.seed, ip);
+ continue;
+ }
+ } catch(final MalformedURLException e) {
+ final String cause = "malformed peer URL";
log.info("publish: disconnected " + this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' from " + this.seed.getIPs() + ": " + cause);
Network.this.sb.peers.peerActions.interfaceDeparture(this.seed, ip);
- continue;
- }
+ continue;
+ }
// success! we have published our peer to a senior peer
// update latest news from the other peer
log.info("publish: handshaked "+ this.seed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getIPs());
diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java
index 94f9db8ed..8f5345681 100644
--- a/source/net/yacy/peers/Protocol.java
+++ b/source/net/yacy/peers/Protocol.java
@@ -58,7 +58,6 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -138,32 +137,37 @@ public final class Protocol {
*/
public static class Post {
- public byte[] result; // contains the result from a successful post or null if no attempt was successful
- public Set<String> unsuccessfulAddresses; // contains a set of addresses which had been tested for submission was without success
- public String successfulAddress; // contains the address which had been successfully used or null if no success with any Address
+ /** Contains the result from a successful post or null if no attempt was successful */
+ private byte[] result;
+ /**
+ * @param targetBaseURL the base target URL
+ * @param targetHash the hash of the target peer
+ * @param path the path on the base URL
+ * @param parts the body content
+ * @param timeout the timeout in milliseconds
+ * @param httpFallback when true, retry as http when a https request failed
+ * @throws IOException
+ */
public Post(
- final String targetAddress,
- final String targetPeerHash,
+ final MultiProtocolURL targetBaseURL,
+ final String targetHash,
final String path,
final Map<String, ContentBody> parts,
final int timeout) throws IOException {
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
httpClient.setTimout(timeout);
- this.result = httpClient.POSTbytes(
- new MultiProtocolURL("http://" + targetAddress + path),
- Seed.b64Hash2hexHash(targetPeerHash) + ".yacyh",
- parts,
- false, true);
- this.unsuccessfulAddresses = new HashSet<>();
- if (this.result == null) {
- this.unsuccessfulAddresses.add(targetAddress);
- this.successfulAddress = null;
- } else {
- this.successfulAddress = targetAddress;
- }
+ MultiProtocolURL targetURL = new MultiProtocolURL(targetBaseURL, path);
+ this.result = httpClient.POSTbytes(targetURL, Seed.b64Hash2hexHash(targetHash) + ".yacyh", parts, false,
+ true);
}
-
+
+ /**
+ * @return the result from a successful post or null if no attempt was successful
+ */
+ public byte[] getResult() {
+ return this.result;
+ }
}
/**
@@ -182,7 +186,7 @@ public final class Protocol {
public static Map<String, String> hello(
final Seed mySeed,
final PeerActions peerActions,
- final String targetAddress,
+ final MultiProtocolURL targetBaseURL,
final String targetHash) {
Map<String, String> result = null;
@@ -202,7 +206,7 @@ public final class Protocol {
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000);
content =
httpClient.POSTbytes(
- new MultiProtocolURL("http://" + targetAddress + "/yacy/hello.html"),
+ new MultiProtocolURL(targetBaseURL, "/yacy/hello.html"),
Seed.b64Hash2hexHash(targetHash) + ".yacyh",
parts,
false, true);
@@ -212,7 +216,7 @@ public final class Protocol {
if ( Thread.currentThread().isInterrupted() ) {Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' interrupted.");
return null;
}
- Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + targetAddress + "; exception: " + e.getMessage());
+ Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + targetBaseURL + "; exception: " + e.getMessage());
// try again (go into loop)
result = null;
}
@@ -222,7 +226,7 @@ public final class Protocol {
+ ((result == null) ? "result null" : ("result=" + result.toString())));
return null;
}
- Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + targetAddress + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + responseTime + " milliseconds");
+ Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "' contacted peer at " + targetBaseURL + ", received " + ((content == null) ? "null" : content.length) + " bytes, time = " + responseTime + " milliseconds");
// check consistency with expectation
Seed otherPeer = null;
@@ -233,7 +237,7 @@ public final class Protocol {
} else {
try {
// patch the remote peer address to avoid that remote peers spoof the network with wrong addresses
- String host = Domains.stripToHostName(targetAddress);
+ String host = targetBaseURL.getHost();
InetAddress ie = Domains.dnsResolve(host);
otherPeer = Seed.genRemoteSeed(seed, false, ie.getHostAddress());
if ( !otherPeer.hash.equals(targetHash) ) {
@@ -342,7 +346,7 @@ public final class Protocol {
} else {
try {
if ( i == 1 ) {
- String host = Domains.stripToHostName(targetAddress);
+ String host = targetBaseURL.getHost();
InetAddress ia = Domains.dnsResolve(host);
if (ia == null) continue;
host = ia.getHostAddress(); // the actual address of the target as we had been successful when contacting them is patched here
@@ -364,23 +368,24 @@ public final class Protocol {
return result;
}
- public static long[] queryRWICount(final String targetAddress, final String targetHash, int timeout) {
+ public static long[] queryRWICount(final MultiProtocolURL targetBaseURL, final Seed target, int timeout) {
// prepare request
final String salt = crypt.randomSalt();
// send request
try {
- final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt);
+ final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
parts.put("object", UTF8.StringBody("rwicount"));
parts.put("env", UTF8.StringBody(""));
- //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetAddress);
- final Post post = new Post(targetAddress, targetHash, "/yacy/query.html", parts, timeout);
- //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetAddress + (post.result == null ? "NULL" : (": length = " + post.result.length)));
+ //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetBaseURL);
+ final Post post = new Post(targetBaseURL, target.hash, "/yacy/query.html", parts, timeout);
+
+ //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetBaseURL + (post.result == null ? "NULL" : (": length = " + post.result.length)));
final Map<String, String> result = FileUtils.table(post.result);
if (result == null || result.isEmpty()) return new long[] {-1, -1};
- //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetAddress + " : result = " + result.toString());
+ //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetBaseURL + " : result = " + result.toString());
final String resp = result.get("response");
- //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetAddress + " : response = " + resp);
+ //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetBaseURL + " : response = " + resp);
if (resp == null) return new long[] {-1, -1};
String magic = result.get("magic");
if (magic == null) magic = "0";
@@ -390,7 +395,7 @@ public final class Protocol {
return new long[] {-1, -1};
}
} catch (final Exception e ) {
- //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetAddress + ": " + e.getMessage());
+ //ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetBaseURL + ": " + e.getMessage());
if (Network.log.isFine()) Network.log.fine("yacyClient.queryRWICount error:" + e.getMessage());
return new long[] {-1, -1};
}
@@ -518,7 +523,8 @@ public final class Protocol {
Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");
if(targetBaseURL.startsWith("https")) {
/* First mark https unavailable on this peer before removing any interface */
- target.setFlagSSLAvailable(false);
+ target.setFlagSSLAvailable(false);
+ event.peers.updateConnected(target);
} else {
event.peers.peerActions.interfaceDeparture(target, ip);
}
@@ -608,7 +614,8 @@ public final class Protocol {
Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");
if(targetBaseURL.startsWith("https")) {
/* First mark https unavailable on this peer before removing any interface */
- target.setFlagSSLAvailable(false);
+ target.setFlagSSLAvailable(false);
+ event.peers.updateConnected(target);
} else {
event.peers.peerActions.interfaceDeparture(target, ip);
}
@@ -1222,11 +1229,11 @@ public final class Protocol {
if(targetBaseURL.startsWith("https")) {
/* First mark https unavailable on this peer before removing anything else */
target.setFlagSSLAvailable(false);
+ event.peers.updateConnected(target);
} else {
target.setFlagSolrAvailable(false);
}
}
- target.setFlagSolrAvailable(false || myseed);
return -1;
}
} catch(InterruptedException e) {
@@ -1477,26 +1484,34 @@ public final class Protocol {
return true;
}
- public static Map<String, String> permissionMessage(final String targetAddress, final String targetHash) {
- // ask for allowed message size and attachment size
- // if this replies null, the peer does not answer
-
+ /**
+ * Post a request asking for allowed message size and attachment size to the
+ * target peer on the selected target ip. All parameters must not be null.
+ *
+ * @param targetBaseURL
+ * the public base URL of the target peer on one of its reported IP
+ * addresses in {@link Seed#getIPs()}
+ * @param target
+ * the target peer
+ * @param sb
+ * the switchboard instance
+ * @return the result of the request
+ * @throws IOException
+ * when the peer doesn't answer on this IP or any other error
+ * occurred
+ */
+ public static Map<String, String> permissionMessage(final MultiProtocolURL targetBaseURL, final Seed target,
+ final Switchboard sb) throws IOException {
// prepare request
final String salt = crypt.randomSalt();
// send request
- try {
- final Map<String, ContentBody> parts =
- basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt);
- parts.put("process", UTF8.StringBody("permission"));
- final Post post = new Post(targetAddress, targetAddress, "/yacy/message.html", parts, 6000);
- final Map<String, String> result = FileUtils.table(post.result);
- return result;
- } catch (final Exception e ) {
- // most probably a network time-out exception
- Network.log.warn("yacyClient.permissionMessage error:" + e.getMessage());
- return null;
- }
+ final Map<String, ContentBody> parts = basicRequestParts(sb, target.hash, salt);
+ parts.put("process", UTF8.StringBody("permission"));
+ final Post post = new Post(targetBaseURL, target.hash, "/yacy/message.html", parts, 6000);
+
+ final Map<String, String> result = FileUtils.table(post.result);
+ return result;
}
public static Map<String, String> crawlReceipt(
@@ -1702,12 +1717,21 @@ public final class Protocol {
final ReferenceContainerCache<WordReference> indexes,
boolean gzipBody,
final int timeout) {
- for (String ip : targetSeed.getIPs()) {
+ final boolean preferHttps = Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED, SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);
+ for (final String ip : targetSeed.getIPs()) {
if (ip == null) {
Network.log.warn("no address for transferRWI");
return null;
}
- final String address = targetSeed.getPublicAddress(ip);
+ MultiProtocolURL targetBaseURL = null;
+ try {
+ targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, preferHttps);
+ } catch(final MalformedURLException e) {
+ Network.log.info("yacyClient.transferRWI malformed target URL : " + targetBaseURL);
+ // disconnect unavailable peer ip
+ Switchboard.getSwitchboard().peers.peerActions.interfaceDeparture(targetSeed, ip);
+ continue;
+ }
// prepare post values
final String salt = crypt.randomSalt();
@@ -1746,12 +1770,25 @@ public final class Protocol {
parts.put("entryc", UTF8.StringBody(Integer.toString(indexcount)));
parts.put("indexes", UTF8.StringBody(entrypost.toString()));
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout);
- final byte[] content =
- httpClient.POSTbytes(
- new MultiProtocolURL("http://" + address + "/yacy/transferRWI.html"),
- targetSeed.getHexHash() + ".yacyh",
- parts,
- gzipBody, true);
+ byte[] content = null;
+ try {
+ content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),
+ targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);
+ } catch(final IOException e) {
+ if(targetBaseURL.isHTTPS()) {
+ targetBaseURL = targetSeed.getPublicMultiprotocolURL(ip, false);
+ /* Failed with https : retry with http on the same address */
+ content = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL, "/yacy/transferRWI.html"),
+ targetSeed.getHexHash() + ".yacyh", parts, gzipBody, true);
+ if(content != null) {
+ /* Success with http : mark SSL as unavailable on the target peer */
+ Network.log.info("yacyClient.transferRWI SSL unavailable on address " + ip);
+ Switchboard.getSwitchboard().peers.updateConnected(targetSeed);
+ }
+ } else {
+ throw e;
+ }
+ }
final Iterator<String> v = FileUtils.strings(content);
// this should return a list of urlhashes that are unknown
@@ -1761,7 +1798,7 @@ public final class Protocol {
result.put(Seed.IP, ip); // add used ip to result for error handling (in case no "result" key was received)
return result;
} catch (final Exception e ) {
- Network.log.info("yacyClient.transferRWI to " + address + " error: " + e.getMessage());
+ Network.log.info("yacyClient.transferRWI to " + targetBaseURL + " error: " + e.getMessage());
// disconnect unavailable peer ip
Switchboard.getSwitchboard().peers.peerActions.interfaceDeparture(targetSeed, ip);
}
@@ -1908,7 +1945,10 @@ public final class Protocol {
final Map<String, ContentBody> parts =
basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
parts.put("object", UTF8.StringBody("host"));
- final Post post = new Post(target.getPublicAddress(target.getIP()), target.hash, "/yacy/idx.json", parts, 30000);
+ final String remoteBaseURL = target.getPublicURL(target.getIP(),
+ Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
+ SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT));
+ final Post post = new Post(new MultiProtocolURL(remoteBaseURL), target.hash, "/yacy/idx.json", parts, 30000);
if ( post.result == null || post.result.length == 0 ) {
Network.log.warn("yacyClient.loadIDXHosts error: empty result");
return null;
diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java
index 5ec1ce52c..562b0eb93 100644
--- a/source/net/yacy/peers/Seed.java
+++ b/source/net/yacy/peers/Seed.java
@@ -68,6 +68,7 @@ import net.yacy.cora.date.AbstractFormatter;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
+import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.yacy.Distribution;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.order.Digest;
@@ -791,7 +792,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
* @return an URL string for the given peer ip
* @throws RuntimeException when the ip parameter is null
*/
- public final String getPublicURL(final String ip, final boolean preferHTTPS) {
+ public final String getPublicURL(final String ip, final boolean preferHTTPS) throws RuntimeException {
if (ip == null) {
throw new RuntimeException("ip == NULL"); // that should not happen in Peer-to-Peer mode (but can in Intranet mode)
}
@@ -822,6 +823,27 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
}
return sb.toString();
}
+
+ /**
+ * Generate a public URL Multiprotocol instance using a given ip. This combines
+ * the ip with the http(s) port and encloses the ip with square brackets if the
+ * ip is of typeIPv6
+ *
+ * @param ip
+ * a host name or ip address
+ * @param preferHTTPS
+ * when true and https is available on this Seed, use it as the
+ * scheme part of the url
+ * @return an MultiProtocolURL instance for the given peer ip
+ * @throws RuntimeException
+ * when the ip parameter is null
+ * @throws MalformedURLException
+ * when the ip and port could not make a well formed URL
+ */
+ public final MultiProtocolURL getPublicMultiprotocolURL(final String ip, final boolean preferHTTPS)
+ throws RuntimeException, MalformedURLException {
+ return new MultiProtocolURL(getPublicURL(ip, preferHTTPS));
+ }
/** @return the port number of this seed or <code>-1</code> if not present */
public final int getPort() {
diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java
index 88860ef77..71b91cab9 100644
--- a/source/net/yacy/search/SwitchboardConstants.java
+++ b/source/net/yacy/search/SwitchboardConstants.java
@@ -520,6 +520,13 @@ public final class SwitchboardConstants {
public static final String NETWORK_BOOTSTRAP_SEEDLIST_STUB = "network.unit.bootstrap.seedlist";
public static final String NETWORK_SEARCHVERIFY = "network.unit.inspection.searchverify";
+
+ /** Key of the setting controlling whether https should be preferred for in-protocol operations when available on remote peers.
+ * A distinct general setting is available to control whether https sould be used for remote search queries : see {@link #REMOTESEARCH_HTTPS_PREFERRED} */
+ public static final String NETWORK_PROTOCOL_HTTPS_PREFERRED = "network.unit.protocol.https.preferred";
+
+ /** Default setting value controlling whether https should be preferred for in-protocol operations when available on remote peers */
+ public static final boolean NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT = false;
/**
* appearance