Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Improved MultiprocotolURL non ASCII characters support.
luccioman2f191e0e1cc1
diff --git a/source/net/yacy/cora/document/id/MultiProtocolURL.java b/source/net/yacy/cora/document/id/MultiProtocolURL.javaindex effa2b550..b0cf9c7dd 100644--- a/source/net/yacy/cora/document/id/MultiProtocolURL.java+++ b/source/net/yacy/cora/document/id/MultiProtocolURL.java@@ -37,6 +37,7 @@ import java.io.UnsupportedEncodingException;import java.net.InetAddress;import java.net.MalformedURLException;import java.net.URLDecoder;+import java.nio.charset.StandardCharsets;import java.util.BitSet;import java.util.LinkedHashMap;import java.util.LinkedHashSet;@@ -184,7 +185,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU// identify protocolurl = url.trim();-+if (url.startsWith("//")) {// patch for urls starting with "//" which can be found in the wildurl = "http:" + url;@@ -680,12 +681,20 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolUreturn sbuf;}+ /**+ * Decodes a <code>application/x-www-form-urlencoded</code> string using UTF-8 encoding.+ *+ * @param s the string to decode+ * @return the newly decoded string+ */public static String unescape(final String s) {- try {- return URLDecoder.decode(s, "UTF-8");- } catch (UnsupportedEncodingException e) {- return null; // unreachable- }+ try {+ return URLDecoder.decode(s, StandardCharsets.UTF_8.name());+ } catch (UnsupportedEncodingException e) {+ /* This should not happen */+ ConcurrentLog.logException(e);+ return s;+ }}private void identPort(final String inputURL, final int dflt) throws MalformedURLException {@@ -887,15 +896,10 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolUreturn ((this.port >= 0) && (this.host != null)) ? this.host + ":" + this.port : ((this.host != null) ? this.host : "");}+ /**+ * @return the host part of this URL, Punycode encoded for Internationalized Domain Names+ */public String getHost() {- /*- if (this.host == null) return null;- if (this.host.length() > 0 && this.host.charAt(0) == '[') {- int p = this.host.indexOf(']');- if (p < 0) return this.host;- return this.host.substring(1, p);- }- */return this.host;}@@ -975,8 +979,9 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU}/**- * Tokenized url as string (without the protocol)- * @return example "host com path file ext"+ * Tokenizes url as string (without the protocol).+ * For example "http://host.com/path/file.txt" returns "host com path file ext"+ * @return url tokens as one string*/public String toTokens() {return toTokens(unescape(this.urlstub(true,true)));@@ -993,7 +998,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolUchar c;for (int i = 0; i < s.length(); i++) {c = s.charAt(i);- if ((c >= '0' && c <='9') || (c >= 'a' && c <='z') || (c >= 'A' && c <='Z')) sb.append(c); else sb.append(' ');+ if (Character.isAlphabetic(c) || Character.isDigit(c)) sb.append(c); else sb.append(' ');}// split the string into tokens and add all camel-case splitting@@ -1059,8 +1064,8 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU}private static CharType charType(final char c) {- if (c >= 'a' && c <= 'z') return CharType.low;- if (c >= '0' && c <= '9') return CharType.number;+ if (Character.isLowerCase(c)) return CharType.low;+ if (Character.isDigit(c)) return CharType.number;return CharType.high;}diff --git a/test/java/net/yacy/cora/document/id/MultiProtocolURLTest.java b/test/java/net/yacy/cora/document/id/MultiProtocolURLTest.javaindex 3236d84f6..596afed8a 100644--- a/test/java/net/yacy/cora/document/id/MultiProtocolURLTest.java+++ b/test/java/net/yacy/cora/document/id/MultiProtocolURLTest.java@@ -3,6 +3,7 @@ package net.yacy.cora.document.id;import static org.junit.Assert.*;import java.net.MalformedURLException;+import java.nio.charset.StandardCharsets;import java.util.HashMap;import java.util.LinkedHashMap;import java.util.Map;@@ -131,7 +132,14 @@ public class MultiProtocolURLTest {new String[]{"//www.yacy.net:?query=test", "www.yacy.net"},new String[]{"http://www.yacy.net?data=1/2/3", "www.yacy.net"},- new String[]{"http://www.yacy.net?url=http://test.com", "www.yacy.net"}+ new String[]{"http://www.yacy.net?url=http://test.com", "www.yacy.net"},+ /* Punycode encoded internationalized domain name : Algeria TLD */+ new String[]{"http://xn--ggbdmbaav3cjl1c9heugfv.xn--lgbbat1ad8j/", "xn--ggbdmbaav3cjl1c9heugfv.xn--lgbbat1ad8j"},+ /* Internationalized domain name : Algeria TLD */+ new String[]{"http://مركزأسماءالنطاقات.الجزائر/", "xn--ggbdmbaav3cjl1c9heugfv.xn--lgbbat1ad8j"},+ /* Internationalized domain name : Chinese Ministry of education */+ new String[]{"http://教育部.中国/", "xn--wcvs22dzol.xn--fiqs8s"},+ /*http://教育部.中国/ */};for (int i = 0; i < testStrings.length; i++) {@@ -182,7 +190,9 @@ public class MultiProtocolURLTest {new String[]{"http://www.heise.de/newsticker/thema/%23saukontrovers", "http://www.heise.de/newsticker/thema/%23saukontrovers"}, // http://mantis.tokeek.de/view.php?id=519new String[]{"http://www.heise.de/newsticker/thema/#saukontrovers", "http://www.heise.de/newsticker/thema/"}, // anchor fragmentnew String[]{"http://www.liferay.com/community/wiki/-/wiki/Main/Wiki+Portlet", "http://www.liferay.com/community/wiki/-/wiki/Main/Wiki+Portlet"}, // http://mantis.tokeek.de/view.php?id=559- new String[]{"http://de.wikipedia.org/wiki/Philippe_Ariès", "http://de.wikipedia.org/wiki/Philippe_Ari%C3%A8s"} // UTF-8 2 byte char+ new String[]{"http://de.wikipedia.org/wiki/Philippe_Ariès", "http://de.wikipedia.org/wiki/Philippe_Ari%C3%A8s"}, // UTF-8 2 byte char+ new String[] {"https://zh.wikipedia.org/wiki/Wikipedia:方針與指引", "https://zh.wikipedia.org/wiki/Wikipedia:%E6%96%B9%E9%87%9D%E8%88%87%E6%8C%87%E5%BC%95"}, // UTF-8 3 bytes chars+ new String[] {"http://教育部.中国/jyb_xwfb/", "http://xn--wcvs22dzol.xn--fiqs8s/jyb_xwfb/"} // Internationalized Domain Name};for (String[] testString : testStrings) {@@ -204,8 +214,23 @@ public class MultiProtocolURLTest {public void testGetAttribute() throws Exception {// some test url/uri with problems in the pastString[][] testStrings = new String[][]{- // teststring , expectedresult- new String[]{"http://yacy.net?&test", "test"}+ // teststring , expectedresultkey, expectedresultvalue+ new String[]{"http://yacy.net?&test", "test", ""},+ /* Encoded UTF-8 2 bytes characters parameter value */+ new String[]{"https://zh.wikipedia.org/w/index.php?search=encodedlatinchars%C3%A0%C3%A4%C3%A2%C3%A9%C3%A8%C3%AF%C3%AE%C3%B4%C3%B6%C3%B9", "search", "encodedlatincharsàäâéèïîôöù"},+ /* Non encoded UTF-8 2 bytes characters parameter value */+ new String[]{"http://yacy.net?query=unencodedlatincharsàäâéèïîôöù", "query", "unencodedlatincharsàäâéèïîôöù"},+ /* Encoded UTF-8 3 bytes characters parameter value */+ new String[]{"https://zh.wikipedia.org/w/index.php?query=%E6%96%B9%E9%87%9D%E8%88%87%E6%8C%87%E5%BC%95", "query", "方針與指引"},+ /* Non encoded UTF-8 3 bytes characters parameter value */+ new String[]{"https://zh.wikipedia.org/w/index.php?query=方針與指引", "query", "方針與指引"},+ /* Non encoded rfc3986 unreserved ascii chars parameter value */+ new String[]{"https://example.net?query=-.~_", "query", "-.~_"},+ /* Encoded rfc3986 reserved ascii chars parameter value */+ new String[]{"https://example.net?query=%3A%2F%3F%23%40%24%26%2B%2C%3B%3D", "query", ":/?#@$&+,;="},+ /* Non-Encoded rfc3986 reserved ascii chars parameter value+ * (some reserved characters have a meaning here and can not be passed as non-encoded without breaking the parameter value : #, &, +) */+ new String[]{"https://example.net?query=:/?[]@!$'()*,;=", "query", ":/?[]@!$'()*,;="},};for (String[] testString : testStrings) {@@ -214,10 +239,10 @@ public class MultiProtocolURLTest {String shouldBe = testString[1];MultiProtocolURL resultUrl = new MultiProtocolURL(testString[0]);+ System.out.println(" -> " + resultUrl.toNormalform(false));Map<String, String> attr = resultUrl.getAttributes();- assertEquals("", attr.get(shouldBe));- System.out.println(" -> " + resultUrl.toNormalform(false));+ assertEquals(testString[2], attr.get(shouldBe));}}@@ -242,10 +267,10 @@ public class MultiProtocolURLTest {}/**- * Test of toTokens method, of class MultiProtocolURL.+ * Test of toTokens static method, of class MultiProtocolURL.*/@Test- public void testToTokens() {+ public void testStaticToTokens() {// test string pairs which should generate equal resultsString[][] testString = new String[][]{{"abc", "abc "},@@ -261,6 +286,94 @@ public class MultiProtocolURLTest {assertEquals("input: "+s[0]+"="+s[1],result1, result2);}}++ /**+ * Unit tests for {@link MultiProtocolURL#toTokens()}+ * @throws MalformedURLException when+ */+ @Test+ public void testToTokens() throws MalformedURLException {+ String[][] testStrings = new String[][]{+ // test string , "expected tokens"+ new String[]{"http://yacy.net?&test", "yacy net test"},+ new String[]{"http://example.net/camelCased/subpath1/PATH_EXAMPLE", "example net camelCased subpath1 PATH EXAMPLE camel Cased subpath 1"},+ /* Encoded UTF-8 2 bytes characters parameter value */+ new String[]{"https://zh.wikipedia.org/w/index.php?search=encodedlatinchars%C3%A0%C3%A4%C3%A2%C3%A9%C3%A8%C3%AF%C3%AE%C3%B4%C3%B6%C3%B9", "zh wikipedia org w index php search encodedlatincharsàäâéèïîôöù"},+ /* Non encoded UTF-8 2 bytes characters parameter value */+ new String[]{"http://yacy.net?query=unencodedlatincharsàäâéèïîôöù", "yacy net query unencodedlatincharsàäâéèïîôöù"},+ /* Encoded UTF-8 3 bytes characters parameter value */+ new String[]{"https://zh.wikipedia.org/w/index.php?query=%E6%96%B9%E9%87%9D%E8%88%87%E6%8C%87%E5%BC%95", "zh wikipedia org w index php query 方針與指引"},+ /* Non encoded UTF-8 3 bytes characters parameter value */+ new String[]{"https://zh.wikipedia.org/w/index.php?query=方針與指引", "zh wikipedia org w index php query 方針與指引"},+ /* Non encoded rfc3986 unreserved ascii chars parameter value */+ new String[]{"https://example.net?query=-.~_", "example net query"},+ /* Encoded rfc3986 reserved ascii chars parameter value */+ new String[]{"https://example.net?query=%3A%2F%3F%23%40%24%26%2B%2C%3B%3D", "example net query"},+ /* Non-Encoded rfc3986 reserved ascii chars parameter value+ * (some reserved characters have a meaning here and can not be passed as non-encoded without breaking the parameter value : #, &, +) */+ new String[]{"https://example.net?query=:/?[]@!$'()*,;=", "example net query"}+ };++ for (int i = 0; i < testStrings.length; i++) {+ String[] testString = testStrings[i];+ MultiProtocolURL resultUrl = new MultiProtocolURL(testString[0]);+ String tokens = resultUrl.toTokens();+ assertEquals("Test toTokens : " + i, testString[1], tokens);+ }+ }++ /**+ * Unit tests for {@link MultiProtocolURL#escape(String)}+ */+ @Test+ public void testEscape() {+ String[] testStrings = {+ "",+ "asciiString",+ "latin chars:àäâéèïîôöù",+ "logograms:正體字/繁體字",+ "with spaces and\ttab",+ "rfc3986 unreserved ascii chars:-.~_",+ "rfc3986 reserved ascii chars::/?#[]@!$&'()*+,;=",+ "http://simpleurl.com/",+ "http://urlwithqueryandanchor.net/path?q=asciiquery&p1=param1&p2=pâräm2&p3=简化字#anchor" };+ for (String testString : testStrings) {+ String encoded = MultiProtocolURL.escape(testString).toString();+ assertTrue("Encoded string contains only ascii chars",+ StandardCharsets.US_ASCII.newEncoder().canEncode(encoded));+ assertEquals("escape/unescape consistency", testString,+ MultiProtocolURL.unescape(encoded));+ }+ }++ /**+ * Unit tests for {@link MultiProtocolURL#unescape(String)}+ */+ @Test+ public void testUnescape() {+ String[][] testStrings = new String[][] {+ // test string , "expected unencoded result"+ new String[] { "", "" }, new String[] { "asciiString", "asciiString" },+ new String[] { "encoded latinchars : %C3%A0%C3%A4%C3%A2%C3%A9%C3%A8%C3%AF%C3%AE%C3%B4%C3%B6%C3%B9",+ "encoded latinchars : àäâéèïîôöù" },+ new String[] { "unencoded latin chars : àäâéèïîôöù", "unencoded latin chars : àäâéèïîôöù" },+ new String[] { "encoded logograms : %E6%96%B9%E9%87%9D%E8%88%87%E6%8C%87%E5%BC%95",+ "encoded logograms : 方針與指引" },+ new String[] { "unencoded logograms : 方針與指引", "unencoded logograms : 方針與指引" },+ new String[] { "with spaces and\ttab", "with spaces and\ttab" },+ new String[] { "unencoded rfc3986 unreserved ascii chars:-.~_",+ "unencoded rfc3986 unreserved ascii chars:-.~_" },+ new String[] { "http://simpleurl.com/", "http://simpleurl.com/" },+ new String[] {+ "http://url-with-unencoded-query-and-anchor.net/path?q=asciiquery&p1=param1&p2=pâräm2&p3=简化字#anchor",+ "http://url-with-unencoded-query-and-anchor.net/path?q=asciiquery&p1=param1&p2=pâräm2&p3=简化字#anchor" }, };+ for (int i = 0; i < testStrings.length; i++) {+ String[] testString = testStrings[i];+ String unescaped = MultiProtocolURL.unescape(testString[0]);+ assertEquals(testString[1], unescaped);+ }+ }+}