Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

Made possible to use https for remote search on peers with SSL enabled.

lucciomanf01aac31fdfd
diff --git a/defaults/yacy.init b/defaults/yacy.init
index eeddf3f97..6976ac940 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -925,6 +925,8 @@ remotesearch.result.store=true
remotesearch.result.store.maxsize=-1
remotesearch.maxload.rwi=8.0
remotesearch.maxload.solr=4.0
+# When available on the target peer, whether https should be preferred to perform remote search
+remotesearch.https.preferred = false
# Control whether remote Solr instances responses should be binary encoded. Responses are transferred as XML when set to false.
remote.solr.binaryResponse.enabled=true
diff --git a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java
index b8622b1e1..defce133c 100644
--- a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java
+++ b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java
@@ -28,15 +28,9 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
-import net.yacy.cora.document.id.MultiProtocolURL;
-import net.yacy.cora.protocol.Domains;
-import net.yacy.cora.protocol.HeaderFramework;
-import net.yacy.cora.util.CommonPattern;
-import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.kelondro.util.MemoryControl;
-import net.yacy.search.schema.CollectionSchema;
-import net.yacy.search.schema.WebgraphSchema;
+import javax.net.ssl.SSLContext;
+import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
@@ -48,18 +42,35 @@ import org.apache.http.HttpResponseInterceptor;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
+import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.GzipDecompressingEntity;
+import org.apache.http.conn.scheme.PlainSocketFactory;
+import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.conn.scheme.SchemeRegistry;
+import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
+import org.apache.http.conn.ssl.SSLSocketFactory;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
+import org.apache.http.ssl.SSLContextBuilder;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
+import net.yacy.cora.document.id.MultiProtocolURL;
+import net.yacy.cora.protocol.Domains;
+import net.yacy.cora.protocol.HeaderFramework;
+import net.yacy.cora.util.CommonPattern;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.kelondro.util.MemoryControl;
+import net.yacy.search.schema.CollectionSchema;
+import net.yacy.search.schema.WebgraphSchema;
+
@SuppressWarnings("deprecation")
public class RemoteInstance implements SolrInstance {
private String solrurl;
- private final Object client; // not declared as org.apache.http.impl.client.DefaultHttpClient to avoid warnings during compilation. TODO: switch to org.apache.http.impl.client.HttpClientBuilder
+ private final HttpClient client;
private final String defaultCoreName;
private final ConcurrentUpdateSolrClient defaultServer;
private final Collection<String> coreNames;
@@ -129,63 +140,108 @@ public class RemoteInstance implements SolrInstance {
}
}
if (solraccount.length() > 0) {
- org.apache.http.impl.conn.PoolingClientConnectionManager cm = new org.apache.http.impl.conn.PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager
- cm.setMaxTotal(100);
- cm.setDefaultMaxPerRoute(100);
-
- this.client = new org.apache.http.impl.client.DefaultHttpClient(cm) {
- @Override
- protected HttpContext createHttpContext() {
- HttpContext context = super.createHttpContext();
- AuthCache authCache = new org.apache.http.impl.client.BasicAuthCache();
- BasicScheme basicAuth = new BasicScheme();
- HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol());
- authCache.put(targetHost, basicAuth);
- context.setAttribute(org.apache.http.client.protocol.HttpClientContext.AUTH_CACHE, authCache);
- this.setHttpRequestRetryHandler(new org.apache.http.impl.client.DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry
- return context;
- }
- };
- org.apache.http.params.HttpParams params = ((org.apache.http.impl.client.DefaultHttpClient) this.client).getParams();
- org.apache.http.params.HttpConnectionParams.setConnectionTimeout(params, timeout);
- org.apache.http.params.HttpConnectionParams.setSoTimeout(params, timeout);
- ((org.apache.http.impl.client.DefaultHttpClient) this.client).addRequestInterceptor(new HttpRequestInterceptor() {
- @Override
- public void process(final HttpRequest request, final HttpContext context) throws IOException {
- if (!request.containsHeader(HeaderFramework.ACCEPT_ENCODING)) request.addHeader(HeaderFramework.ACCEPT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP);
- if (!request.containsHeader(HTTP.CONN_DIRECTIVE)) request.addHeader(HTTP.CONN_DIRECTIVE, "close"); // prevent CLOSE_WAIT
- }
-
- });
- ((org.apache.http.impl.client.DefaultHttpClient) this.client).addResponseInterceptor(new HttpResponseInterceptor() {
- @Override
- public void process(final HttpResponse response, final HttpContext context) throws IOException {
- HttpEntity entity = response.getEntity();
- if (entity != null) {
- Header ceheader = entity.getContentEncoding();
- if (ceheader != null) {
- HeaderElement[] codecs = ceheader.getElements();
- for (HeaderElement codec : codecs) {
- if (codec.getName().equalsIgnoreCase(HeaderFramework.CONTENT_ENCODING_GZIP)) {
- response.setEntity(new GzipDecompressingEntity(response.getEntity()));
- return;
- }
- }
- }
- }
- }
- });
- org.apache.http.impl.client.BasicCredentialsProvider credsProvider = new org.apache.http.impl.client.BasicCredentialsProvider();
- credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
- ((org.apache.http.impl.client.DefaultHttpClient) this.client).setCredentialsProvider(credsProvider);
+ /* Note : optionally trusting self-signed certificate on an external remote Solr may be considered for convenience */
+ this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, false);
+ } else if(u.isHTTPS()){
+ /* Here we must trust self-signed certificates as most peers with SSL enabled use such certificates */
+ this.client = buildCustomHttpClient(timeout, u, solraccount, solrpw, host, true);
} else {
- this.client = null;
+ // The default HttpSolrClient will be used
+ this.client = null;
}
this.defaultServer = (ConcurrentUpdateSolrClient) getServer(this.defaultCoreName);
if (this.defaultServer == null) throw new IOException("cannot connect to url " + url + " and connect core " + defaultCoreName);
}
+ /**
+ * @param solraccount eventual user name used to authenticate on the target Solr
+ * @param solraccount eventual password used to authenticate on the target Solr
+ * @param trustSelfSignedCertificates when true, https connections to an host rpviding a self-signed certificate are accepted
+ * @return a new apache HttpClient instance usable as a custom http client by SolrJ
+ */
+ private static HttpClient buildCustomHttpClient(final int timeout, final MultiProtocolURL u, final String solraccount, final String solrpw,
+ final String host, final boolean trustSelfSignedCertificates) {
+
+ /* Important note : deprecated use of Apache classes is required because SolrJ still use them internally (see HttpClientUtil).
+ * Upgrade only when Solr implementation will become compatible */
+
+ org.apache.http.impl.conn.PoolingClientConnectionManager cm;
+ SchemeRegistry registry = null;
+ if(trustSelfSignedCertificates) {
+ SSLContext sslContext;
+ try {
+ sslContext = SSLContextBuilder.create().loadTrustMaterial(TrustSelfSignedStrategy.INSTANCE).build();
+ registry = new SchemeRegistry();
+ registry.register(
+ new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
+ registry.register(
+ new Scheme("https", 443, new SSLSocketFactory(sslContext, AllowAllHostnameVerifier.INSTANCE)));
+ } catch (final Exception e) {
+ // Should not happen
+ ConcurrentLog.warn("RemoteInstance", "Error when initializing SSL context trusting self-signed certificates.", e);
+ registry = null;
+ }
+ }
+ if(registry != null) {
+ cm = new org.apache.http.impl.conn.PoolingClientConnectionManager(registry);
+ } else {
+ cm = new org.apache.http.impl.conn.PoolingClientConnectionManager(); // try also: ThreadSafeClientConnManager
+ }
+ cm.setMaxTotal(100);
+ cm.setDefaultMaxPerRoute(100);
+
+ org.apache.http.impl.client.DefaultHttpClient result = new org.apache.http.impl.client.DefaultHttpClient(cm) {
+ @Override
+ protected HttpContext createHttpContext() {
+ HttpContext context = super.createHttpContext();
+ AuthCache authCache = new org.apache.http.impl.client.BasicAuthCache();
+ BasicScheme basicAuth = new BasicScheme();
+ HttpHost targetHost = new HttpHost(u.getHost(), u.getPort(), u.getProtocol());
+ authCache.put(targetHost, basicAuth);
+ context.setAttribute(org.apache.http.client.protocol.HttpClientContext.AUTH_CACHE, authCache);
+ this.setHttpRequestRetryHandler(new org.apache.http.impl.client.DefaultHttpRequestRetryHandler(0, false)); // no retries needed; we expect connections to fail; therefore we should not retry
+ return context;
+ }
+ };
+ org.apache.http.params.HttpParams params = result.getParams();
+ org.apache.http.params.HttpConnectionParams.setConnectionTimeout(params, timeout);
+ org.apache.http.params.HttpConnectionParams.setSoTimeout(params, timeout);
+ result.addRequestInterceptor(new HttpRequestInterceptor() {
+ @Override
+ public void process(final HttpRequest request, final HttpContext context) throws IOException {
+ if (!request.containsHeader(HeaderFramework.ACCEPT_ENCODING)) request.addHeader(HeaderFramework.ACCEPT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP);
+ if (!request.containsHeader(HTTP.CONN_DIRECTIVE)) request.addHeader(HTTP.CONN_DIRECTIVE, "close"); // prevent CLOSE_WAIT
+ }
+
+ });
+ result.addResponseInterceptor(new HttpResponseInterceptor() {
+ @Override
+ public void process(final HttpResponse response, final HttpContext context) throws IOException {
+ HttpEntity entity = response.getEntity();
+ if (entity != null) {
+ Header ceheader = entity.getContentEncoding();
+ if (ceheader != null) {
+ HeaderElement[] codecs = ceheader.getElements();
+ for (HeaderElement codec : codecs) {
+ if (codec.getName().equalsIgnoreCase(HeaderFramework.CONTENT_ENCODING_GZIP)) {
+ response.setEntity(new GzipDecompressingEntity(response.getEntity()));
+ return;
+ }
+ }
+ }
+ }
+ }
+ });
+ if(solraccount != null && !solraccount.isEmpty()) {
+ org.apache.http.impl.client.BasicCredentialsProvider credsProvider = new org.apache.http.impl.client.BasicCredentialsProvider();
+ credsProvider.setCredentials(new AuthScope(host, AuthScope.ANY_PORT), new UsernamePasswordCredentials(solraccount, solrpw));
+ result.setCredentialsProvider(credsProvider);
+ }
+
+ return result;
+ }
+
@Override
public int hashCode() {
return this.solrurl.hashCode();
@@ -228,22 +284,29 @@ public class RemoteInstance implements SolrInstance {
ConcurrentUpdateSolrClient s = this.server.get(name);
if (s != null) return s;
// create new http server
+ final MultiProtocolURL u;
+ try {
+ u = new MultiProtocolURL(this.solrurl + name);
+ } catch (final MalformedURLException e) {
+ return null;
+ }
if (this.client != null) {
- final MultiProtocolURL u;
- try {
- u = new MultiProtocolURL(this.solrurl + name);
- } catch (final MalformedURLException e) {
- return null;
- }
- String host = u.getHost();
- int port = u.getPort();
- String solrpath = u.getPath();
- String p = "http://" + host + ":" + port + solrpath;
- ConcurrentLog.info("RemoteSolrConnector", "connecting Solr authenticated with url:" + p);
- s = new ConcurrentUpdateSolrClient(p, ((org.apache.http.impl.client.DefaultHttpClient) this.client), 10, Runtime.getRuntime().availableProcessors());
+ final String solrServerURL;
+ if(StringUtils.isNotEmpty(u.getUserInfo())) {
+ /* Remove user authentication info from the URL, as authentication will be handled by the custom http client */
+ String host = u.getHost();
+ int port = u.getPort();
+ String solrpath = u.getPath();
+ solrServerURL = u.getProtocol() + "://" + host + ":" + port + solrpath;
+ ConcurrentLog.info("RemoteSolrConnector", "connecting Solr authenticated with url : " + u);
+ } else {
+ solrServerURL = u.toString();
+ ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url : " + u);
+ }
+ s = new ConcurrentUpdateSolrClient(solrServerURL, this.client, 10, Runtime.getRuntime().availableProcessors());
} else {
- ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url:" + this.solrurl + name);
- s = new ConcurrentUpdateSolrClient(this.solrurl + name, queueSizeByMemory(), Runtime.getRuntime().availableProcessors());
+ ConcurrentLog.info("RemoteSolrConnector", "connecting Solr with url : " + this.solrurl + name);
+ s = new ConcurrentUpdateSolrClient(u.toString(), queueSizeByMemory(), Runtime.getRuntime().availableProcessors());
}
//s.setAllowCompression(true);
s.setSoTimeout(this.timeout);
diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java
index 793dc13f4..94f9db8ed 100644
--- a/source/net/yacy/peers/Protocol.java
+++ b/source/net/yacy/peers/Protocol.java
@@ -487,8 +487,14 @@ public final class Protocol {
SearchResult result = null;
for (String ip: target.getIPs()) {
//if (ip.indexOf(':') >= 0) System.out.println("Search target: IPv6: " + ip);
- String clusteraddress = target.getPublicAddress(ip);
- if (target.clash(event.peers.mySeed().getIPs())) clusteraddress = "localhost:" + event.peers.mySeed().getPort();
+ final String targetBaseURL;
+ if (target.clash(event.peers.mySeed().getIPs())) {
+ targetBaseURL = "http://localhost:" + event.peers.mySeed().getPort();
+ } else {
+ targetBaseURL = target.getPublicURL(ip,
+ Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
+ SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT));
+ }
try {
result =
new SearchResult(
@@ -504,13 +510,18 @@ public final class Protocol {
maxDistance,
partitions,
target.getHexHash() + ".yacyh",
- clusteraddress,
+ targetBaseURL,
secondarySearchSuperviser
);
break;
} catch (final IOException e ) {
Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");
- event.peers.peerActions.interfaceDeparture(target, ip);
+ if(targetBaseURL.startsWith("https")) {
+ /* First mark https unavailable on this peer before removing any interface */
+ target.setFlagSSLAvailable(false);
+ } else {
+ event.peers.peerActions.interfaceDeparture(target, ip);
+ }
return -1;
}
}
@@ -571,6 +582,9 @@ public final class Protocol {
event.addExpectedRemoteReferences(count);
SearchResult result = null;
for (String ip: target.getIPs()) {
+ final String targetBaseURL = target.getPublicURL(ip,
+ Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
+ SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT));
try {
result =
new SearchResult(
@@ -586,13 +600,18 @@ public final class Protocol {
maxDistance,
partitions,
target.getHexHash() + ".yacyh",
- target.getPublicAddress(ip),
+ targetBaseURL,
null
);
break;
} catch (final IOException e ) {
Network.log.info("SEARCH failed, Peer: " + target.hash + ":" + target.getName() + " (" + e.getMessage() + ")");
- event.peers.peerActions.interfaceDeparture(target, ip);
+ if(targetBaseURL.startsWith("https")) {
+ /* First mark https unavailable on this peer before removing any interface */
+ target.setFlagSSLAvailable(false);
+ } else {
+ event.peers.peerActions.interfaceDeparture(target, ip);
+ }
return -1;
}
}
@@ -845,7 +864,7 @@ public final class Protocol {
final int maxDistance,
final int partitions,
final String hostname,
- final String hostaddress,
+ final String targetBaseURL,
final SecondarySearchSuperviser secondarySearchSuperviser
) throws IOException {
// send a search request to peer with remote Hash
@@ -902,8 +921,7 @@ public final class Protocol {
}
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 8000);
- //System.out.println("Protocol: http://" + hostaddress + "/yacy/search.html" + requestPartsToString(parts)); // DEBUG
- byte[] a = httpClient.POSTbytes(new MultiProtocolURL("http://" + hostaddress + "/yacy/search.html"), hostname, parts, false, true);
+ byte[] a = httpClient.POSTbytes(new MultiProtocolURL(targetBaseURL + "/yacy/search.html"), hostname, parts, false, true);
if (a != null && a.length > 200000) {
// there is something wrong. This is too large, maybe a hack on the other side?
a = null;
@@ -986,8 +1004,8 @@ public final class Protocol {
/** The solr query to run */
private final SolrQuery solrQuery;
- /** The instance address */
- private final String address;
+ /** The instance base URL */
+ private final String targetBaseURL;
/** The target seed information */
private final Seed target;
@@ -1010,15 +1028,15 @@ public final class Protocol {
/**
* Constructor. All parameters are required to not be null.
* @param solrQuery the Solr query to run
- * @param address the instance address : host name or IP + the eventual port
+ * @param targetBaseURL the instance base URL : http(s) + host name or IP + the eventual port
* @param target the remote target seed information
* @param timeout the request timeout in milliseconds
*/
- protected SolrRequestTask(final SolrQuery solrQuery, final String address, final Seed target,
+ protected SolrRequestTask(final SolrQuery solrQuery, final String targetBaseURL, final Seed target,
final boolean mySeed, final int timeout, final QueryResponse[] rsp, final SolrDocumentList[] docList) {
super("Protocol.solrQuery(" + solrQuery.getQuery() + " to " + target.hash + ")");
this.solrQuery = solrQuery;
- this.address = address;
+ this.targetBaseURL = targetBaseURL;
this.target = target;
this.mySeed = mySeed;
this.timeout = timeout;
@@ -1040,13 +1058,13 @@ public final class Protocol {
/* Strip too large details to avoid polluting this log with complete remote stack traces */
message = message.substring(0, MAX_ERROR_MESSAGE_LENGTH) + "...";
}
- log.fine(messageBegin + " at " + this.address + " : " + message);
+ log.fine(messageBegin + " at " + this.targetBaseURL + " : " + message);
}
@Override
public void run() {
try {
- this.instance = new RemoteInstance("http://" + this.address, null, "solr", this.timeout); // this is a 'patch configuration' which considers 'solr' as default collection
+ this.instance = new RemoteInstance(this.targetBaseURL, null, "solr", this.timeout); // this is a 'patch configuration' which considers 'solr' as default collection
try {
boolean useBinaryResponseWriter = SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT;
if (Switchboard.getSwitchboard() != null) {
@@ -1170,13 +1188,20 @@ public final class Protocol {
} else {
try {
final boolean myseed = target == event.peers.mySeed();
+ final String targetBaseURL;
+ if(myseed) {
+ targetBaseURL = "http://localhost:" + target.getPort();
+ } else {
+ targetBaseURL = target.getPublicURL(ip,
+ Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
+ SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT));
+ }
if (!myseed && !target.getFlagSolrAvailable()) { // skip if peer.dna has flag that last try resulted in error
Network.log.info("SEARCH skip (solr), remote Solr interface not accessible, peer=" + target.getName());
return -1;
}
- final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress(ip);
final int solrtimeout = Switchboard.getSwitchboard().getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 6000);
- SolrRequestTask remoteRequest = new SolrRequestTask(solrQuery, address, target, myseed, solrtimeout, rsp, docList);
+ SolrRequestTask remoteRequest = new SolrRequestTask(solrQuery, targetBaseURL, target, myseed, solrtimeout, rsp, docList);
remoteRequest.start();
remoteRequest.join(solrtimeout); // just wait until timeout appears
if (remoteRequest.isAlive()) {
@@ -1190,6 +1215,20 @@ public final class Protocol {
target.setFlagSolrAvailable(false || myseed);
return -1; // give up, leave remoteRequest abandoned.
}
+
+ if (rsp[0] == null || docList[0] == null) {
+ Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " returned null");
+ if(!myseed) {
+ if(targetBaseURL.startsWith("https")) {
+ /* First mark https unavailable on this peer before removing anything else */
+ target.setFlagSSLAvailable(false);
+ } else {
+ target.setFlagSolrAvailable(false);
+ }
+ }
+ target.setFlagSolrAvailable(false || myseed);
+ return -1;
+ }
} catch(InterruptedException e) {
/* Current thread might be interrupted by SearchEvent.cleanup() :
* we must not in that case mark the target as not available but rather transmit the exception to the caller (likely RemoteSearch.solrRemoteSearch) */
@@ -1201,12 +1240,6 @@ public final class Protocol {
}
}
- if (rsp[0] == null || docList[0] == null) {
- Network.log.info("SEARCH failed (solr), remote Peer: " + target.getName() + "/" + target.getPublicAddress(ip) + " returned null");
- target.setFlagSolrAvailable(false || localsearch);
- return -1;
- }
-
// evaluate facets
if(useSolrFacets) {
for (String field: event.query.facetfields) {
diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java
index b1703e246..5ec1ce52c 100644
--- a/source/net/yacy/peers/Seed.java
+++ b/source/net/yacy/peers/Seed.java
@@ -756,10 +756,11 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
}
/**
- * generate a public address using a given ip. This combines the ip with the port and encloses the ip
+ * generate a public address using a given ip. This combines the ip with the http port and encloses the ip
* with square brackets if the ip is of typeIPv6
* @param ip
* @return an address string which can be used as host:port part of an url (if no port avail returns just host)
+ * @ŧhrows RuntimeException when the ip parameter is null
*/
public final String getPublicAddress(final String ip) {
if (ip == null) throw new RuntimeException("ip == NULL"); // that should not happen in Peer-to-Peer mode (but can in Intranet mode)
@@ -781,6 +782,46 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
}
return sb.toString();
}
+
+ /**
+ * Generate a public URL using a given ip. This combines the ip with the http(s) port and encloses the ip
+ * with square brackets if the ip is of typeIPv6
+ * @param ip a host name or ip address
+ * @param preferHTTPS when true and https is available on this Seed, use it as the scheme part of the url
+ * @return an URL string for the given peer ip
+ * @throws RuntimeException when the ip parameter is null
+ */
+ public final String getPublicURL(final String ip, final boolean preferHTTPS) {
+ if (ip == null) {
+ throw new RuntimeException("ip == NULL"); // that should not happen in Peer-to-Peer mode (but can in Intranet mode)
+ }
+ final String scheme;
+ final String port;
+ if(preferHTTPS && getFlagSSLAvailable()) {
+ scheme = "https://";
+ port = this.dna.get(Seed.PORTSSL);
+ } else {
+ scheme = "http://";
+ port = this.dna.get(Seed.PORT);
+ }
+ final StringBuilder sb = new StringBuilder(scheme.length() + ip.length() + 8); // / = surplus for port
+ sb.append(scheme);
+ if (ip.indexOf(':') >= 0) {
+ if (!ip.startsWith("[")) sb.append('[');
+ sb.append(ip);
+ if (!ip.endsWith("]")) sb.append(']');
+ } else {
+ sb.append(ip);
+ }
+ if (port == null || port.length() < 2 || port.length() > 5) {
+ //just skip port if peer didn't report it..... may finally depart
+ Network.log.severe(preferHTTPS ? "https" : "http " + " port not wellformed for peer" + this.getName() + ": " + port == null ? "null" : port);
+ } else {
+ sb.append(':');
+ sb.append(port);
+ }
+ return sb.toString();
+ }
/** @return the port number of this seed or <code>-1</code> if not present */
public final int getPort() {
diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java
index e5d2374fe..88860ef77 100644
--- a/source/net/yacy/search/SwitchboardConstants.java
+++ b/source/net/yacy/search/SwitchboardConstants.java
@@ -290,6 +290,12 @@ public final class SwitchboardConstants {
/** Default maximum system load allowing remote Solr searches */
public static final float REMOTESEARCH_MAXLOAD_SOLR_DEFAULT = 4.0f;
+ /** Key of the setting controlling whether https should be preferred for remote searches, when available on the target peer */
+ public static final String REMOTESEARCH_HTTPS_PREFERRED = "remotesearch.https.preferred";
+
+ /** Default setting value controlling whether https should be preferred for remote searches, when available on the target peer */
+ public static final boolean REMOTESEARCH_HTTPS_PREFERRED_DEFAULT = false;
+
/**
* Setting key to configure whether responses from remote Solr instances
* should be binary encoded :