Apache Nutch is an extensible and scalable web crawler
NUTCH-3206 XML parsers should not be vulnerable to XXE attacks (#954)
Lewis John McGibbney773b4321ca25
diff --git a/src/java/org/apache/nutch/exchange/Exchanges.java b/src/java/org/apache/nutch/exchange/Exchanges.javaindex 1e0518b6b..37b2406bf 100644--- a/src/java/org/apache/nutch/exchange/Exchanges.java+++ b/src/java/org/apache/nutch/exchange/Exchanges.java@@ -22,6 +22,7 @@ import org.apache.nutch.plugin.Extension;import org.apache.nutch.plugin.ExtensionPoint;import org.apache.nutch.plugin.PluginRepository;import org.apache.nutch.plugin.PluginRuntimeException;+import org.apache.nutch.util.XmlUtil;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import org.w3c.dom.Element;@@ -104,7 +105,7 @@ public class Exchanges {final List<ExchangeConfig> configList = new LinkedList<>();try {- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();+ DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory();DocumentBuilder builder = factory.newDocumentBuilder();Element rootElement = builder.parse(inputSource).getDocumentElement();NodeList exchangeList = rootElement.getElementsByTagName("exchange");diff --git a/src/java/org/apache/nutch/indexer/IndexWriters.java b/src/java/org/apache/nutch/indexer/IndexWriters.javaindex f8ae8ee86..4b809d1e4 100644--- a/src/java/org/apache/nutch/indexer/IndexWriters.java+++ b/src/java/org/apache/nutch/indexer/IndexWriters.java@@ -26,6 +26,7 @@ import org.apache.nutch.plugin.ExtensionPoint;import org.apache.nutch.plugin.PluginRepository;import org.apache.nutch.plugin.PluginRuntimeException;import org.apache.nutch.util.NutchConfiguration;+import org.apache.nutch.util.XmlUtil;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import org.w3c.dom.Document;@@ -121,7 +122,7 @@ public class IndexWriters {InputSource inputSource = new InputSource(ssInputStream);try {- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();+ DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory();DocumentBuilder builder = factory.newDocumentBuilder();Document document = builder.parse(inputSource);Element rootElement = document.getDocumentElement();diff --git a/src/java/org/apache/nutch/parse/ParsePluginsReader.java b/src/java/org/apache/nutch/parse/ParsePluginsReader.javaindex 978e3c097..8b84d4479 100644--- a/src/java/org/apache/nutch/parse/ParsePluginsReader.java+++ b/src/java/org/apache/nutch/parse/ParsePluginsReader.java@@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory;import org.apache.hadoop.conf.Configuration;import org.apache.nutch.util.NutchConfiguration;+import org.apache.nutch.util.XmlUtil;/*** A reader to load the information stored in the@@ -102,7 +103,7 @@ class ParsePluginsReader {inputSource = new InputSource(ppInputStream);try {- factory = DocumentBuilderFactory.newInstance();+ factory = XmlUtil.newSecureDocumentBuilderFactory();parser = factory.newDocumentBuilder();document = parser.parse(inputSource);} catch (Exception e) {diff --git a/src/java/org/apache/nutch/plugin/PluginManifestParser.java b/src/java/org/apache/nutch/plugin/PluginManifestParser.javaindex 95208fa43..dca202910 100644--- a/src/java/org/apache/nutch/plugin/PluginManifestParser.java+++ b/src/java/org/apache/nutch/plugin/PluginManifestParser.java@@ -31,6 +31,7 @@ import javax.xml.parsers.DocumentBuilderFactory;import javax.xml.parsers.ParserConfigurationException;import org.apache.hadoop.conf.Configuration;+import org.apache.nutch.util.XmlUtil;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import org.w3c.dom.Document;@@ -158,7 +159,7 @@ public class PluginManifestParser {*/private Document parseXML(URL url)throws SAXException, IOException, ParserConfigurationException {- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();+ DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory();DocumentBuilder builder = factory.newDocumentBuilder();return builder.parse(url.openStream());}diff --git a/src/java/org/apache/nutch/tools/DmozParser.java b/src/java/org/apache/nutch/tools/DmozParser.javaindex 4924e3b1b..5de457fb3 100644--- a/src/java/org/apache/nutch/tools/DmozParser.java+++ b/src/java/org/apache/nutch/tools/DmozParser.java@@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.io.MD5Hash;import org.apache.nutch.util.NutchConfiguration;+import org.apache.nutch.util.XmlUtil;import org.apache.xerces.util.XMLChar;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -296,9 +297,7 @@ public class DmozParser {boolean includeAdult, int skew, Pattern topicPattern)throws IOException, SAXException, ParserConfigurationException {- SAXParserFactory parserFactory = SAXParserFactory.newInstance();- parserFactory.setFeature("http://xml.org/sax/features/external-general-entities", false);- parserFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);+ SAXParserFactory parserFactory = XmlUtil.newSecureSAXParserFactory();SAXParser parser = parserFactory.newSAXParser();XMLReader reader = parser.getXMLReader();reader.setFeature("http://xml.org/sax/features/external-general-entities", false);diff --git a/src/java/org/apache/nutch/util/DomUtil.java b/src/java/org/apache/nutch/util/DomUtil.javaindex 50cc43c98..c31c2a1f0 100644--- a/src/java/org/apache/nutch/util/DomUtil.java+++ b/src/java/org/apache/nutch/util/DomUtil.java@@ -60,6 +60,7 @@ public class DomUtil {InputSource input;try {+ XmlUtil.configureSecure(parser);input = new InputSource(is);input.setEncoding("UTF-8");parser.parse(input);@@ -87,9 +88,10 @@ public class DomUtil {public static void saveDom(OutputStream os, Element e) {DOMSource source = new DOMSource(e);- TransformerFactory transFactory = TransformerFactory.newInstance();+ TransformerFactory transFactory;Transformer transformer;try {+ transFactory = XmlUtil.newSecureTransformerFactory();transformer = transFactory.newTransformer();transformer.setOutputProperty("indent", "yes");transformer.setOutputProperty(OutputKeys.ENCODING,diff --git a/src/java/org/apache/nutch/util/XmlUtil.java b/src/java/org/apache/nutch/util/XmlUtil.javanew file mode 100644index 000000000..a8ce9b64f--- /dev/null+++ b/src/java/org/apache/nutch/util/XmlUtil.java@@ -0,0 +1,135 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.nutch.util;++import javax.xml.XMLConstants;+import javax.xml.parsers.DocumentBuilderFactory;+import javax.xml.parsers.ParserConfigurationException;+import javax.xml.parsers.SAXParserFactory;+import javax.xml.transform.TransformerConfigurationException;+import javax.xml.transform.TransformerFactory;++import org.apache.xerces.parsers.DOMParser;+import org.xml.sax.SAXException;+import org.xml.sax.SAXNotRecognizedException;+import org.xml.sax.SAXNotSupportedException;++/**+ * Helpers that return XML parser/transformer factories configured to reject+ * XXE (external entity) attacks. Prefer these over bare+ * {@link DocumentBuilderFactory#newInstance()} /+ * {@link SAXParserFactory#newInstance()} /+ * {@link TransformerFactory#newInstance()}.+ */+public final class XmlUtil {++ private static final String DISALLOW_DOCTYPE_DECL =+ "http://apache.org/xml/features/disallow-doctype-decl";+ private static final String EXTERNAL_GENERAL_ENTITIES =+ "http://xml.org/sax/features/external-general-entities";+ private static final String EXTERNAL_PARAMETER_ENTITIES =+ "http://xml.org/sax/features/external-parameter-entities";+ private static final String LOAD_EXTERNAL_DTD =+ "http://apache.org/xml/features/nonvalidating/load-external-dtd";++ private XmlUtil() {+ }++ /**+ * @return a {@link DocumentBuilderFactory} hardened against XXE+ * @throws ParserConfigurationException if a required secure feature is+ * unsupported+ */+ public static DocumentBuilderFactory newSecureDocumentBuilderFactory()+ throws ParserConfigurationException {+ return newSecureDocumentBuilderFactory(false);+ }++ /**+ * @param namespaceAware whether the factory should be namespace-aware+ * @return a {@link DocumentBuilderFactory} hardened against XXE+ * @throws ParserConfigurationException if a required secure feature is+ * unsupported+ */+ public static DocumentBuilderFactory newSecureDocumentBuilderFactory(+ boolean namespaceAware) throws ParserConfigurationException {+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();+ factory.setNamespaceAware(namespaceAware);+ factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);+ factory.setFeature(DISALLOW_DOCTYPE_DECL, true);+ factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false);+ factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);+ factory.setFeature(LOAD_EXTERNAL_DTD, false);+ factory.setXIncludeAware(false);+ // Sonar java:S2755 "Going the extra mile" — does not alone block XXE+ factory.setExpandEntityReferences(false);+ return factory;+ }++ /**+ * @return a {@link SAXParserFactory} hardened against XXE+ * @throws ParserConfigurationException if a required secure feature is+ * unsupported+ * @throws SAXNotRecognizedException if a feature name is not recognized+ * @throws SAXNotSupportedException if a feature value is not supported+ */+ public static SAXParserFactory newSecureSAXParserFactory()+ throws ParserConfigurationException, SAXNotRecognizedException,+ SAXNotSupportedException {+ SAXParserFactory factory = SAXParserFactory.newInstance();+ factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);+ factory.setFeature(DISALLOW_DOCTYPE_DECL, true);+ factory.setFeature(EXTERNAL_GENERAL_ENTITIES, false);+ factory.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);+ factory.setXIncludeAware(false);+ return factory;+ }++ /**+ * @return a {@link TransformerFactory} that cannot load external DTDs or+ * stylesheets+ * @throws TransformerConfigurationException if secure attributes cannot be+ * applied+ */+ public static TransformerFactory newSecureTransformerFactory()+ throws TransformerConfigurationException {+ TransformerFactory factory = TransformerFactory.newInstance();+ factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);+ factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");+ factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");+ return factory;+ }++ /**+ * Apply XXE-hardening features to an Xerces {@link DOMParser}.+ *+ * @param parser parser to configure+ * @throws SAXException if a secure feature cannot be set+ */+ public static void configureSecure(DOMParser parser) throws SAXException {+ parser.setFeature(DISALLOW_DOCTYPE_DECL, true);+ parser.setFeature(EXTERNAL_GENERAL_ENTITIES, false);+ parser.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);+ parser.setFeature(LOAD_EXTERNAL_DTD, false);+ try {+ parser.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);+ } catch (SAXNotRecognizedException | SAXNotSupportedException e) {+ // Xerces' DOMParser does not expose the JAXP secure-processing feature.+ // DTDs and external entities are already disabled above.+ }+ }+}diff --git a/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java b/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.javaindex 030c6bf66..ee4e22ec2 100644--- a/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java+++ b/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java@@ -26,6 +26,7 @@ import org.apache.nutch.parse.ParseException;import org.apache.nutch.parse.ParseResult;import org.apache.nutch.parse.ParseStatus;import org.apache.nutch.parse.ParseText;+import org.apache.nutch.util.XmlUtil;import org.apache.hadoop.conf.Configuration;import org.slf4j.Logger;@@ -45,6 +46,7 @@ import java.util.HashMap;import javax.xml.parsers.DocumentBuilder;import javax.xml.parsers.DocumentBuilderFactory;+import javax.xml.parsers.ParserConfigurationException;import org.xml.sax.InputSource;@@ -167,11 +169,14 @@ public class CCParseFilter implements HtmlParseFilter {}}- /** Configure a namespace aware XML parser. */- private static final DocumentBuilderFactory FACTORY = DocumentBuilderFactory- .newInstance();+ /** Configure a namespace aware XML parser hardened against XXE. */+ private static final DocumentBuilderFactory FACTORY;static {- FACTORY.setNamespaceAware(true);+ try {+ FACTORY = XmlUtil.newSecureDocumentBuilderFactory(true);+ } catch (ParserConfigurationException e) {+ throw new ExceptionInInitializerError(e);+ }}/** Creative Commons' namespace URI. */diff --git a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.javaindex ebeb652dd..6d66b9b9d 100644--- a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java+++ b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java@@ -26,7 +26,6 @@ import java.util.HashSet;import java.util.Map;import java.util.Set;-import javax.xml.parsers.DocumentBuilderFactory;import javax.xml.parsers.ParserConfigurationException;import org.xml.sax.SAXException;@@ -55,6 +54,7 @@ import org.apache.nutch.protocol.ProtocolException;import org.apache.nutch.protocol.http.api.HttpBase;import org.apache.hadoop.conf.Configuration;import org.apache.nutch.util.NutchConfiguration;+import org.apache.nutch.util.XmlUtil;/*** <p>@@ -275,7 +275,7 @@ public class Http extends HttpBase {InputStream is = conf.getConfResourceAsInputStream(authFile);if (is != null) {- Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder()+ Document doc = XmlUtil.newSecureDocumentBuilderFactory().newDocumentBuilder().parse(is);Element rootElement = doc.getDocumentElement();diff --git a/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java b/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.javaindex 2aeb722b2..2714e3e19 100644--- a/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java+++ b/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/RegexURLNormalizer.java@@ -32,13 +32,12 @@ import java.util.regex.Matcher;import java.util.regex.Pattern;import java.util.regex.PatternSyntaxException;-import javax.xml.parsers.DocumentBuilderFactory;-import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.nutch.net.URLNormalizer;import org.apache.nutch.net.URLNormalizers;import org.apache.nutch.util.NutchConfiguration;+import org.apache.nutch.util.XmlUtil;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import org.w3c.dom.Document;@@ -224,7 +223,7 @@ public class RegexURLNormalizer extends Configured implements URLNormalizer {try {// borrowed heavily from code in Configuration.java- Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder()+ Document doc = XmlUtil.newSecureDocumentBuilderFactory().newDocumentBuilder().parse(new InputSource(reader));Element root = doc.getDocumentElement();if (!"regex-normalize".equals(root.getTagName())) {diff --git a/src/test/org/apache/nutch/util/TestXmlUtil.java b/src/test/org/apache/nutch/util/TestXmlUtil.javanew file mode 100644index 000000000..d5c9d7dae--- /dev/null+++ b/src/test/org/apache/nutch/util/TestXmlUtil.java@@ -0,0 +1,90 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.nutch.util;++import java.io.ByteArrayInputStream;+import java.nio.charset.StandardCharsets;++import javax.xml.parsers.DocumentBuilder;+import javax.xml.parsers.DocumentBuilderFactory;++import org.junit.jupiter.api.Test;+import org.w3c.dom.Document;+import org.w3c.dom.Element;+import org.xml.sax.SAXException;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertThrows;++/**+ * Unit tests for {@link XmlUtil} XXE hardening.+ */+public class TestXmlUtil {++ @Test+ public void testSecureFactoryParsesSimpleDocument() throws Exception {+ DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory();+ DocumentBuilder builder = factory.newDocumentBuilder();+ Document doc = builder.parse(new ByteArrayInputStream(+ "<root attr=\"ok\"><child/></root>".getBytes(StandardCharsets.UTF_8)));+ assertEquals("root", doc.getDocumentElement().getTagName());+ assertEquals("ok", doc.getDocumentElement().getAttribute("attr"));+ }++ @Test+ public void testSecureFactoryRejectsDoctype() throws Exception {+ DocumentBuilderFactory factory = XmlUtil.newSecureDocumentBuilderFactory();+ DocumentBuilder builder = factory.newDocumentBuilder();+ String xxe = "<?xml version=\"1.0\"?>"+ + "<!DOCTYPE foo [<!ENTITY xxe SYSTEM \"file:///etc/passwd\">]>"+ + "<foo>&xxe;</foo>";+ assertThrows(SAXException.class, () -> builder+ .parse(new ByteArrayInputStream(xxe.getBytes(StandardCharsets.UTF_8))));+ }++ @Test+ public void testDomUtilRejectsDoctype() {+ String xxe = "<?xml version=\"1.0\"?>"+ + "<!DOCTYPE foo [<!ENTITY xxe SYSTEM \"file:///etc/passwd\">]>"+ + "<foo>&xxe;</foo>";+ Element element = DomUtil.getDom(+ new ByteArrayInputStream(xxe.getBytes(StandardCharsets.UTF_8)));+ // DomUtil swallows parse errors and returns null+ assertEquals(null, element);+ }++ @Test+ public void testDomUtilParsesDocumentWithLeadingComment() {+ String xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"+ + "<!-- just a comment -->"+ + "<subcollections><subcollection><id>nutch</id></subcollection>"+ + "</subcollections>";+ Element element = DomUtil.getDom(+ new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)));+ assertNotNull(element);+ assertEquals("subcollections", element.getTagName());+ assertEquals(1, element.getElementsByTagName("subcollection").getLength());+ }++ @Test+ public void testNamespaceAwareFactory() throws Exception {+ DocumentBuilderFactory factory = XmlUtil+ .newSecureDocumentBuilderFactory(true);+ assertNotNull(factory.newDocumentBuilder());+ }+}