Apache Nutch is an extensible and scalable web crawler
NUTCH-3180 BasicURLNormalizer missing catching ICUInputTooLongException (#923)
Luca5bcb0ab87d26
diff --git a/src/java/org/apache/nutch/util/URLUtil.java b/src/java/org/apache/nutch/util/URLUtil.javaindex fd036480a..851d10bef 100644--- a/src/java/org/apache/nutch/util/URLUtil.java+++ b/src/java/org/apache/nutch/util/URLUtil.java@@ -24,6 +24,7 @@ import java.net.URL;import java.util.Locale;import java.util.regex.Pattern;+import com.ibm.icu.util.ICUException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -652,7 +653,6 @@ public class URLUtil {// IndexOutOfBoundsException: thrown (undocumented) if one "label"// (non-ASCII dot-separated segment) is longer than 256 characters,// cf. https://bugs.openjdk.java.net/browse/JDK-6806873- LOG.debug("Failed to convert IDN host {}: ", host, e);throw (MalformedURLException) new MalformedURLException("Invalid IDN " + host + ": " + e.getMessage()).initCause(e);}@@ -675,11 +675,22 @@ public class URLUtil {throws MalformedURLException {final IDNA.Info idnaInfo = new IDNA.Info();final StringBuilder hostConverted = new StringBuilder();- if (toAscii) {- idna.nameToASCII(host, hostConverted, idnaInfo);- } else {- idna.nameToUnicode(host, hostConverted, idnaInfo);+ try {+ if (toAscii) {+ idna.nameToASCII(host, hostConverted, idnaInfo);+ } else {+ idna.nameToUnicode(host, hostConverted, idnaInfo);+ }+ } catch (ICUException | IllegalStateException e) {+ // ICU's UTS46 + Punycode conversion throws these unchecked exceptions:+ // ICUException (incl. ICUInputTooLongException from Punycode.encode on an+ // over-long label), IllegalStateException (Punycode).+ // Convert to MalformedURLException so callers (e.g. BasicURLNormalizer)+ // reject the URL instead of crashing the task.+ throw (MalformedURLException) new MalformedURLException(+ "Invalid IDN host " + host + ": " + e.getMessage()).initCause(e);}+if (idnaInfo.hasErrors()) {StringBuilder msg = new StringBuilder();for (IDNA.Error error : idnaInfo.getErrors()) {diff --git a/src/test/org/apache/nutch/util/TestURLUtil.java b/src/test/org/apache/nutch/util/TestURLUtil.javaindex 200ea59a0..7c7267ce8 100644--- a/src/test/org/apache/nutch/util/TestURLUtil.java+++ b/src/test/org/apache/nutch/util/TestURLUtil.java@@ -444,6 +444,22 @@ public class TestURLUtil {() -> URLUtil.convertIDNA2008("xn--a-ä.pt", false));assertThrows(MalformedURLException.class,() -> URLUtil.convertIDNA2008("xn--a-ä.pt", true));++ assertThrows(MalformedURLException.class, // Thai, 1001 units+ () -> URLUtil.convertIDNA2008("ก".repeat(1001), true));+ assertThrows(MalformedURLException.class, // emoji (surrogate pairs), 1002 units+ () -> URLUtil.convertIDNA2008("😀".repeat(501), true));+ assertThrows(MalformedURLException.class, // CJK Ext-B, 1200 units+ () -> URLUtil.convertIDNA2008(+ new String(Character.toChars(0x20000)).repeat(600), true));++ // boundary: exactly 1000 units does not throw; rejected via LABEL_TOO_LONG+ assertThrows(MalformedURLException.class,+ () -> URLUtil.convertIDNA2008("ก".repeat(1000), true));++ // EMPTY_LABEL: a bare dot (empty label) is rejected via IDNA.Info errors+ assertThrows(MalformedURLException.class,+ () -> URLUtil.convertIDNA2008(".", true));}@Test