Apache Nutch is an extensible and scalable web crawler
NUTCH-3192 Address Yetus detsecrets/blanks and SonarCloud security findings in config templates and DummySSL (#944)
Lewis John McGibbneyf8a072fa1c72
diff --git a/.markdownlint.yaml b/.markdownlint.yamlindex a35fb44e3..0afae7f67 100644--- a/.markdownlint.yaml+++ b/.markdownlint.yaml@@ -5,6 +5,7 @@ MD003: falseMD012: falseMD013:line_length: 200+ tables: falseMD022: falseMD025: falseMD033: falsediff --git a/conf/httpclient-auth.xml.template b/conf/httpclient-auth.xml.templateindex f9763bf68..97eb255da 100644--- a/conf/httpclient-auth.xml.template+++ b/conf/httpclient-auth.xml.template@@ -2,12 +2,12 @@<!--This is the authentication configuration file for protocol-httpclient.Different credentials for different authentication scopes can be- configured in this file. If a set of credentials is configured for a+ configured in this file. If a set of credentials is configured for aparticular authentication scope (i.e. particular host, port number,scheme and realm), then that set of credentials would be sent only toservers falling under the specified authentication scope. Apart fromthis at most one set of credentials can be configured as 'default'.-+When authentication is required to fetch a resource from a web-server,the authentication-scope is determined from the host, port, scheme andrealm (if present) obtained from the URL of the page and the@@ -30,14 +30,15 @@the following example.Example:-- <credentials username="susam" password="masus">+ <!-- Also set the password attribute on each credentials element. -->+ <credentials username="YOUR_USERNAME"><default realm="sso"/><authscope host="192.168.101.33" port="80" realm="login"/><authscope host="example" port="8080" realm="blogs"/><authscope host="example" port="8080" realm="wiki"/><authscope host="example" port="80" realm="quiz" scheme="NTLM"/></credentials>- <credentials username="admin" password="nimda">+ <credentials username="YOUR_USERNAME"><authscope host="example" port="8080"/></credentials>@@ -59,8 +60,8 @@support can be located at https://cwiki.apache.org/confluence/display/NUTCH/HttpAuthenticationSchemesHTTP-POST Authentication Support- Http Form-based Authentication is a very common used authentication- mechanism to protect web resources. We extend the 'auth-configuration'+ Http Form-based Authentication is a very common used authentication+ mechanism to protect web resources. We extend the 'auth-configuration'to include information about http form authentication properties as shownin the following example:@@ -71,9 +72,9 @@loginRedirect="true"><loginPostData><field name="ctl00$MainContent$LoginUser$UserName"- value="admin"/>+ value="YOUR_USERNAME"/><field name="ctl00$MainContent$LoginUser$Password"- value="admin123"/>+ value="YOUR_PASSWORD"/></loginPostData><additionalPostHeaders><field name="User-Agent"@@ -86,25 +87,25 @@<policy>BROWSER_COMPATIBILITY</policy></loginCookie></credentials>-+it is critical that the following fields are substituted:* loginUrl - the URL containing the actual <form>* loginFormId - the <form id="$formId" attribute value(or the 'name' attribute if no form is referenced by 'id' attribute)* loginRedirect - if http post login returns redirect code: 301 or 302,and value is true, Http Client will automatically follow the redirect.- * <field name="ctl00$MainContent$LoginUser$UserName" value="admin"+ * <field name="ctl00$MainContent$LoginUser$UserName" value="YOUR_USERNAME"- the <input name"name" and user defined username value used to representthe field and username respectively- * <field name="ctl00$MainContent$LoginUser$Password" value="admin123"+ * <field name="ctl00$MainContent$LoginUser$Password" value="YOUR_PASSWORD"- the <input name"name" and user defined password value used to representthe field and password respectively* <field name="ctl00$MainContent$LoginUser$RememberMe"/>- form element attributes for which we wish to skip fields- * <policy> value from <loginCookie> is a constant value symbol from+ * <policy> value from <loginCookie> is a constant value symbol fromorg.apache.commons.httpclient.cookie.CookiePolicy, like BROWSER_COMPATIBILITY,DEFAULT, RFC_2109, etc.-+More information on HTTP POST can be located athttps://cwiki.apache.org/confluence/display/NUTCH/HttpPostAuthenticationdiff --git a/conf/index-writers.xml.template b/conf/index-writers.xml.templateindex 6ed341cb7..59c2e8072 100644--- a/conf/index-writers.xml.template+++ b/conf/index-writers.xml.template@@ -61,7 +61,7 @@</writer><writer id="indexer_rabbit_1" class="org.apache.nutch.indexwriter.rabbit.RabbitIndexWriter"><parameters>- <param name="server.uri" value="amqp://guest:guest@localhost:5672/"/>+ <param name="server.uri" value="amqp://localhost:5672/"/><param name="binding" value="false"/><param name="binding.arguments" value=""/><param name="exchange.name" value=""/>diff --git a/conf/nutch-default.xml b/conf/nutch-default.xmlindex 0990b9f81..949716dac 100644--- a/conf/nutch-default.xml+++ b/conf/nutch-default.xml@@ -3011,9 +3011,10 @@ one publisher implementation for RabbitMQ (plugin publish-rabbitmq).<property><name>rabbitmq.publisher.server.uri</name>- <value>amqp://guest:guest@localhost:5672/</value>+ <value>amqp://localhost:5672/</value><description>URI with connection parameters in the form+ amqp://hostname:port/virtualHost oramqp://username:password@hostname:port/virtualHostwhere:username is the username for RabbitMQ server.@@ -3021,6 +3022,8 @@ one publisher implementation for RabbitMQ (plugin publish-rabbitmq).hostname is where the RabbitMQ server is running.port is where the RabbitMQ server is listening.virtualHost is where where the exchange is and the user has access.+ A typical local RabbitMQ broker uses the well-known guest/guest+ account; include username:password@ in the URI when required.</description></property>diff --git a/src/plugin/indexer-rabbit/README.md b/src/plugin/indexer-rabbit/README.mdindex 8040cd6c7..a91c19e4b 100644--- a/src/plugin/indexer-rabbit/README.md+++ b/src/plugin/indexer-rabbit/README.md@@ -47,7 +47,7 @@ Each parameter has the form `<param name="<name>" value="<value>"/>` and the parParameter Name | Description | Default value--|--|---server.uri | URI with connection parameters in the form `amqp://<username>:<password>@<hostname>:<port>/<virtualHost>`<br>Where:<ul><li>`<username>` is the username for RabbitMQ server.</li><li>`<password>` is the password for RabbitMQ server.</li><li>`<hostname>` is where the RabbitMQ server is running.</li><li>`<port>` is where the RabbitMQ server is listening.</li><li>`<virtualHost>` is where the exchange is and the user has access.</li></ul> | amqp://guest:guest@localhost:5672/+server.uri | URI with connection parameters in the form `amqp://<hostname>:<port>/<virtualHost>` or `amqp://<username>:<password>@<hostname>:<port>/<virtualHost>`<br>Where:<ul><li>`<username>` is the username for RabbitMQ server.</li><li>`<password>` is the password for RabbitMQ server.</li><li>`<hostname>` is where the RabbitMQ server is running.</li><li>`<port>` is where the RabbitMQ server is listening.</li><li>`<virtualHost>` is where the exchange is and the user has access.</li></ul>A typical local RabbitMQ broker uses the well-known `guest`/`guest` account; include `username:password@` in the URI when required. | amqp://localhost:5672/binding | Whether the relationship between an exchange and a queue is created automatically.<br>**NOTE:** Binding between exchanges is not supported. | falsebinding.arguments | Arguments used in binding. It must have the form `key1=value1,key2=value2`. This value is only used when the exchange's type is headers and the value of binding property is **true**. In other cases is ignored. |exchange.name | Name for the exchange where the messages will be sent. |diff --git a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/DummySSLProtocolSocketFactory.java b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/DummySSLProtocolSocketFactory.javaindex de834f5f6..e0412206a 100644--- a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/DummySSLProtocolSocketFactory.java+++ b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/DummySSLProtocolSocketFactory.java@@ -59,7 +59,7 @@ public class DummySSLProtocolSocketFactory implementsprivate static SSLContext createEasySSLContext() {try {- SSLContext context = SSLContext.getInstance("SSL");+ SSLContext context = SSLContext.getInstance("TLS");context.init(null,new TrustManager[] { new DummyX509TrustManager(null) }, null);return context;