Apache Nutch is an extensible and scalable web crawler
NUTCH-3115 update to set all fields access on each POJO individually, updated JUnit tests, improved logging
Joe Gilvary94a993528fa2
diff --git a/conf/nutch-default.xml b/conf/nutch-default.xmlindex 0f181ffbd..1fddade83 100644--- a/conf/nutch-default.xml+++ b/conf/nutch-default.xml@@ -2343,7 +2343,7 @@ CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when using this</property><property>- <name>index.arbitrary.all.fields.access</name>+ <name>index.arbitrary.all.fields.access.0</name><value></value><description>Whether to pass all indexing filter fields to constructorfor the user POJO. These are the NutchDocument doc, Parse parse,diff --git a/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.java b/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.javaindex 4b6996511..695e01b1c 100644--- a/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.java+++ b/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.java@@ -196,7 +196,12 @@ public class ArbitraryIndexingFilter implements IndexingFilter {} else {theConstructor = theClass.getDeclaredConstructor(String[].class);}- } catch (Exception e) {+ } catch (NoSuchMethodException nme) {+ LOG.error("Exception preparing reflection for constructor. className was {}",+ String.valueOf(className));+ nme.printStackTrace();+ continue;+ } catch (Exception e) {LOG.error("Exception preparing reflection tasks. className was {}",String.valueOf(className));e.printStackTrace();@@ -214,7 +219,6 @@ public class ArbitraryIndexingFilter implements IndexingFilter {inlinks);} else {constrArgs[0] = url.toString();- //System.arraycopy(userConstrArgs,0,constrArgs,1,userConstrArgs.length);instance = theConstructor.newInstance(new Object[]{constrArgs});}@@ -269,8 +273,6 @@ public class ArbitraryIndexingFilter implements IndexingFilter {this.conf = conf;arbitraryAddsCount = conf.getInt("index.arbitrary.function.count",1);LOG.info("Will process the first {} fieldName defs in config.", String.valueOf(arbitraryAddsCount));- allFieldsAccess = conf.getBoolean("index.arbitrary.all.fields.access",false);- LOG.info("POJO classes will " + (allFieldsAccess ? "" : "not ") + "have access to all filter constructor args.");}/**@@ -302,6 +304,8 @@ public class ArbitraryIndexingFilter implements IndexingFilter {userConstrArgs = conf.getTrimmedStrings("index.arbitrary.constructorArgs.".concat(String.valueOf(ndx)));methodName = conf.get("index.arbitrary.methodName.".concat(String.valueOf(ndx)),"");methodArgs = conf.getTrimmedStrings("index.arbitrary.methodArgs.".concat(String.valueOf(ndx)));+ allFieldsAccess = conf.getBoolean("index.arbitrary.all.fields.access.".concat(String.valueOf(ndx)),false);+ LOG.info("POJO class {} will " + (allFieldsAccess ? "" : "not ") + "have access to all filter constructor args.",String.valueOf(className));overwrite = conf.getBoolean("index.arbitrary.overwrite.".concat(String.valueOf(ndx)),false);if (overwrite) {LOG.info("overwrite set == true for processing {}.", fieldName);diff --git a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.java b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.javaindex 79c4e80f3..4b8df2124 100644--- a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.java+++ b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.java@@ -51,7 +51,7 @@ public class PopularityGauge {}public double getPopularityBoost() {- popularityBoost = (Double) doc.getFieldValue("popularityBoost");+ popularityBoost = Double.valueOf(String.valueOf(doc.getFieldValue("popularityBoost")));if (datum.getStatus() == CrawlDatum.STATUS_FETCH_SUCCESS) {Set anchorSet = new HashSet<>(Arrays.asList(inlinks.getAnchors()));if(anchorSet.contains("dinosaur")){@@ -76,10 +76,10 @@ public class PopularityGauge {public static void main(String[] args,NutchDocument doc,Parse parse,- Text url,+ Text url,CrawlDatum datum,Inlinks inlinks) {-+PopularityGauge popGauge = new PopularityGauge(args,doc,parse,url,datum,inlinks);out.println(popGauge.getPopularityBoost());}diff --git a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/TestArbitraryIndexingFilter.java b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/TestArbitraryIndexingFilter.javaindex e62027270..f11b43fa0 100644--- a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/TestArbitraryIndexingFilter.java+++ b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/TestArbitraryIndexingFilter.java@@ -288,7 +288,7 @@ public class TestArbitraryIndexingFilter {public void testAddingNewCalculatedField() throws Exception {conf = NutchConfiguration.create();conf.set("index.arbitrary.function.count","1");- conf.set("index.arbitrary.all.fields.access","true");+ conf.set("index.arbitrary.all.fields.access.0","true");conf.set("index.arbitrary.constructorArgs.0","");conf.set("index.arbitrary.fieldName.0","popularityBoost");conf.set("index.arbitrary.className.0","org.apache.nutch.indexer.arbitrary.PopularityGauge");@@ -301,7 +301,7 @@ public class TestArbitraryIndexingFilter {filter.setConf(conf);doc = new NutchDocument();- Double boostVal = new Double("1.0");+ Double boostVal = Double.valueOf("1.0");doc.add("popularityBoost", boostVal);Assert.assertFalse("doc is empty", doc.getFieldNames().isEmpty());Assert.assertTrue("test if doc has new field with arbitrary value", doc.getField("popularityBoost")@@ -332,8 +332,68 @@ public class TestArbitraryIndexingFilter {Assert.fail(e.getMessage());}-Assert.assertTrue("test if successfully fetched doc has expected value in popularityBoost", doc.getField("popularityBoost").getValues().contains(2.0));}++ /**+ * Test simplest approach to updating POJOs to new signature+ * for constructor+ *+ * @throws Exception+ */+ @Test+ public void testUpdatingPOJOClass() throws Exception {+ conf = NutchConfiguration.create();+ conf.set("index.arbitrary.function.count","4");+ conf.set("index.arbitrary.fieldName.0","foo");+ conf.set("index.arbitrary.className.0","org.apache.nutch.indexer.arbitrary.Echo");+ conf.set("index.arbitrary.constructorArgs.0","Original Echo class added 'bar'");+ conf.set("index.arbitrary.methodName.0","getText");++ conf.set("index.arbitrary.fieldName.1","bogusSite");+ conf.set("index.arbitrary.className.1","org.apache.nutch.indexer.arbitrary.UpdatedEcho");+ conf.set("index.arbitrary.constructorArgs.1","https://www.updatedNutchPluginJunitTest.com");+ conf.set("index.arbitrary.methodName.1","getText");+ conf.set("index.arbitrary.all.fields.access.1","false");++ conf.set("index.arbitrary.fieldName.2","description");+ conf.set("index.arbitrary.className.2","org.apache.nutch.indexer.arbitrary.UpdatedMultiplier");+ conf.set("index.arbitrary.constructorArgs.2","");+ conf.set("index.arbitrary.methodName.2","getProduct");+ conf.set("index.arbitrary.methodArgs.2","-1,3.14");+ conf.set("index.arbitrary.all.fields.access.2","true");++ conf.set("index.arbitrary.fieldName.3","summary");+ conf.set("index.arbitrary.className.3","org.apache.nutch.indexer.arbitrary.UpdatedMultiplier");+ conf.set("index.arbitrary.constructorArgs.3","");+ conf.set("index.arbitrary.methodName.3","getProduct");+ conf.set("index.arbitrary.methodArgs.3","41,25");+ conf.set("index.arbitrary.all.fields.access.3","false");++ filter = new ArbitraryIndexingFilter();+ Assert.assertNotNull("No filter exists for testAddingNewField",filter);++ filter.setConf(conf);+ doc = new NutchDocument();++ try {+ filter.filter(doc, parse, url, crawlDatum, inlinks);+ } catch (Exception e) {+ e.printStackTrace();+ Assert.fail(e.getMessage());+ }++ Assert.assertNotNull(doc);+ Assert.assertFalse("test if doc is not empty", doc.getFieldNames()+ .isEmpty());+ Assert.assertTrue("test if doc still has new field with arbitrary value running with new indexer", doc.getField("foo")+ .getValues().contains("Original Echo class added 'bar'"));+ Assert.assertTrue("test if updated POJO created new field with arbitrary value", doc.getField("bogusSite")+ .getValues().contains("https://www.updatedNutchPluginJunitTest.com"));+ Assert.assertTrue("test updated POJO set new value in existing field", doc.getField("description")+ .getValues().contains("-3.14"));+ Assert.assertTrue("test POJO with both constructor styles supports old calls", doc.getField("summary")+ .getValues().contains("1025.0"));+ }}diff --git a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedEcho.java b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedEcho.javanew file mode 100644index 000000000..0c1e63a58--- /dev/null+++ b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedEcho.java@@ -0,0 +1,59 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.nutch.indexer.arbitrary;++import org.apache.nutch.parse.Parse;+import org.apache.nutch.indexer.NutchDocument;+import org.apache.nutch.crawl.CrawlDatum;+import org.apache.nutch.crawl.Inlinks;+import org.apache.hadoop.io.Text;++import java.io.PrintStream;++public class UpdatedEcho {++ private NutchDocument doc;+ private Parse parse;+ private Text url;+ private CrawlDatum datum;+ private Inlinks inlinks;+ private static PrintStream out = System.out;+ private String words;++ public UpdatedEcho(String args[],+ NutchDocument docIn,+ Parse parseIn,+ Text urlIn,+ CrawlDatum datumIn,+ Inlinks inlinksIn) {+ super();+ doc = docIn;+ parse = parseIn;+ url = urlIn;+ inlinks = inlinksIn;+ datum = datumIn;+ }++ public UpdatedEcho(String args[]) {+ super();+ words = String.valueOf(args[1]);+ }++ public String getText() {+ return words;+ }+}diff --git a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedMultiplier.java b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedMultiplier.javanew file mode 100644index 000000000..165d69351--- /dev/null+++ b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedMultiplier.java@@ -0,0 +1,58 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.nutch.indexer.arbitrary;++import org.apache.nutch.parse.Parse;+import org.apache.nutch.indexer.NutchDocument;+import org.apache.nutch.crawl.CrawlDatum;+import org.apache.nutch.crawl.Inlinks;+import org.apache.hadoop.io.Text;++import java.io.PrintStream;++public class UpdatedMultiplier {+ private float product = 1;+ private static PrintStream err = System.err;+ private static PrintStream out = System.out;+++ public UpdatedMultiplier(String args[]) {+ super();+ }++ public UpdatedMultiplier(String args[],+ NutchDocument doc,+ Parse parse,+ Text url,+ CrawlDatum datum,+ Inlinks inlinks) {+ super();+ }++ public String getProduct(String args[]) {+ int i = args.length - 1;+ try {+ while (i >= 0) {+ product = product * Float.parseFloat(args[i]);+ i--;+ }+ } catch (NumberFormatException nfe) {+ err.println("NumberFormatException while trying to parse " + String.valueOf(args[i]));+ }+ return String.valueOf(product);+ }+}