Skip to content

armgit / nutch

Public Imported from GitHub
0

Apache Nutch is an extensible and scalable web crawler

NUTCH-3115 update to set all fields access on each POJO individually, updated JUnit tests, improved logging

Joe Gilvary94a993528fa2
diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml
index 0f181ffbd..1fddade83 100644
--- a/conf/nutch-default.xml
+++ b/conf/nutch-default.xml
@@ -2343,7 +2343,7 @@ CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when using this
</property>
<property>
- <name>index.arbitrary.all.fields.access</name>
+ <name>index.arbitrary.all.fields.access.0</name>
<value></value>
<description>Whether to pass all indexing filter fields to constructor
for the user POJO. These are the NutchDocument doc, Parse parse,
diff --git a/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.java b/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.java
index 4b6996511..695e01b1c 100644
--- a/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.java
+++ b/src/plugin/index-arbitrary/src/java/org/apache/nutch/indexer/arbitrary/ArbitraryIndexingFilter.java
@@ -196,7 +196,12 @@ public class ArbitraryIndexingFilter implements IndexingFilter {
} else {
theConstructor = theClass.getDeclaredConstructor(String[].class);
}
- } catch (Exception e) {
+ } catch (NoSuchMethodException nme) {
+ LOG.error("Exception preparing reflection for constructor. className was {}",
+ String.valueOf(className));
+ nme.printStackTrace();
+ continue;
+ } catch (Exception e) {
LOG.error("Exception preparing reflection tasks. className was {}",
String.valueOf(className));
e.printStackTrace();
@@ -214,7 +219,6 @@ public class ArbitraryIndexingFilter implements IndexingFilter {
inlinks);
} else {
constrArgs[0] = url.toString();
- //System.arraycopy(userConstrArgs,0,constrArgs,1,userConstrArgs.length);
instance = theConstructor.newInstance(new Object[]{constrArgs});
}
@@ -269,8 +273,6 @@ public class ArbitraryIndexingFilter implements IndexingFilter {
this.conf = conf;
arbitraryAddsCount = conf.getInt("index.arbitrary.function.count",1);
LOG.info("Will process the first {} fieldName defs in config.", String.valueOf(arbitraryAddsCount));
- allFieldsAccess = conf.getBoolean("index.arbitrary.all.fields.access",false);
- LOG.info("POJO classes will " + (allFieldsAccess ? "" : "not ") + "have access to all filter constructor args.");
}
/**
@@ -302,6 +304,8 @@ public class ArbitraryIndexingFilter implements IndexingFilter {
userConstrArgs = conf.getTrimmedStrings("index.arbitrary.constructorArgs.".concat(String.valueOf(ndx)));
methodName = conf.get("index.arbitrary.methodName.".concat(String.valueOf(ndx)),"");
methodArgs = conf.getTrimmedStrings("index.arbitrary.methodArgs.".concat(String.valueOf(ndx)));
+ allFieldsAccess = conf.getBoolean("index.arbitrary.all.fields.access.".concat(String.valueOf(ndx)),false);
+ LOG.info("POJO class {} will " + (allFieldsAccess ? "" : "not ") + "have access to all filter constructor args.",String.valueOf(className));
overwrite = conf.getBoolean("index.arbitrary.overwrite.".concat(String.valueOf(ndx)),false);
if (overwrite) {
LOG.info("overwrite set == true for processing {}.", fieldName);
diff --git a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.java b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.java
index 79c4e80f3..4b8df2124 100644
--- a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.java
+++ b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/PopularityGauge.java
@@ -51,7 +51,7 @@ public class PopularityGauge {
}
public double getPopularityBoost() {
- popularityBoost = (Double) doc.getFieldValue("popularityBoost");
+ popularityBoost = Double.valueOf(String.valueOf(doc.getFieldValue("popularityBoost")));
if (datum.getStatus() == CrawlDatum.STATUS_FETCH_SUCCESS) {
Set anchorSet = new HashSet<>(Arrays.asList(inlinks.getAnchors()));
if(anchorSet.contains("dinosaur")){
@@ -76,10 +76,10 @@ public class PopularityGauge {
public static void main(String[] args,
NutchDocument doc,
Parse parse,
- Text url,
+ Text url,
CrawlDatum datum,
Inlinks inlinks) {
-
+
PopularityGauge popGauge = new PopularityGauge(args,doc,parse,url,datum,inlinks);
out.println(popGauge.getPopularityBoost());
}
diff --git a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/TestArbitraryIndexingFilter.java b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/TestArbitraryIndexingFilter.java
index e62027270..f11b43fa0 100644
--- a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/TestArbitraryIndexingFilter.java
+++ b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/TestArbitraryIndexingFilter.java
@@ -288,7 +288,7 @@ public class TestArbitraryIndexingFilter {
public void testAddingNewCalculatedField() throws Exception {
conf = NutchConfiguration.create();
conf.set("index.arbitrary.function.count","1");
- conf.set("index.arbitrary.all.fields.access","true");
+ conf.set("index.arbitrary.all.fields.access.0","true");
conf.set("index.arbitrary.constructorArgs.0","");
conf.set("index.arbitrary.fieldName.0","popularityBoost");
conf.set("index.arbitrary.className.0","org.apache.nutch.indexer.arbitrary.PopularityGauge");
@@ -301,7 +301,7 @@ public class TestArbitraryIndexingFilter {
filter.setConf(conf);
doc = new NutchDocument();
- Double boostVal = new Double("1.0");
+ Double boostVal = Double.valueOf("1.0");
doc.add("popularityBoost", boostVal);
Assert.assertFalse("doc is empty", doc.getFieldNames().isEmpty());
Assert.assertTrue("test if doc has new field with arbitrary value", doc.getField("popularityBoost")
@@ -332,8 +332,68 @@ public class TestArbitraryIndexingFilter {
Assert.fail(e.getMessage());
}
-
Assert.assertTrue("test if successfully fetched doc has expected value in popularityBoost", doc.getField("popularityBoost")
.getValues().contains(2.0));
}
+
+ /**
+ * Test simplest approach to updating POJOs to new signature
+ * for constructor
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testUpdatingPOJOClass() throws Exception {
+ conf = NutchConfiguration.create();
+ conf.set("index.arbitrary.function.count","4");
+ conf.set("index.arbitrary.fieldName.0","foo");
+ conf.set("index.arbitrary.className.0","org.apache.nutch.indexer.arbitrary.Echo");
+ conf.set("index.arbitrary.constructorArgs.0","Original Echo class added 'bar'");
+ conf.set("index.arbitrary.methodName.0","getText");
+
+ conf.set("index.arbitrary.fieldName.1","bogusSite");
+ conf.set("index.arbitrary.className.1","org.apache.nutch.indexer.arbitrary.UpdatedEcho");
+ conf.set("index.arbitrary.constructorArgs.1","https://www.updatedNutchPluginJunitTest.com");
+ conf.set("index.arbitrary.methodName.1","getText");
+ conf.set("index.arbitrary.all.fields.access.1","false");
+
+ conf.set("index.arbitrary.fieldName.2","description");
+ conf.set("index.arbitrary.className.2","org.apache.nutch.indexer.arbitrary.UpdatedMultiplier");
+ conf.set("index.arbitrary.constructorArgs.2","");
+ conf.set("index.arbitrary.methodName.2","getProduct");
+ conf.set("index.arbitrary.methodArgs.2","-1,3.14");
+ conf.set("index.arbitrary.all.fields.access.2","true");
+
+ conf.set("index.arbitrary.fieldName.3","summary");
+ conf.set("index.arbitrary.className.3","org.apache.nutch.indexer.arbitrary.UpdatedMultiplier");
+ conf.set("index.arbitrary.constructorArgs.3","");
+ conf.set("index.arbitrary.methodName.3","getProduct");
+ conf.set("index.arbitrary.methodArgs.3","41,25");
+ conf.set("index.arbitrary.all.fields.access.3","false");
+
+ filter = new ArbitraryIndexingFilter();
+ Assert.assertNotNull("No filter exists for testAddingNewField",filter);
+
+ filter.setConf(conf);
+ doc = new NutchDocument();
+
+ try {
+ filter.filter(doc, parse, url, crawlDatum, inlinks);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+
+ Assert.assertNotNull(doc);
+ Assert.assertFalse("test if doc is not empty", doc.getFieldNames()
+ .isEmpty());
+ Assert.assertTrue("test if doc still has new field with arbitrary value running with new indexer", doc.getField("foo")
+ .getValues().contains("Original Echo class added 'bar'"));
+ Assert.assertTrue("test if updated POJO created new field with arbitrary value", doc.getField("bogusSite")
+ .getValues().contains("https://www.updatedNutchPluginJunitTest.com"));
+ Assert.assertTrue("test updated POJO set new value in existing field", doc.getField("description")
+ .getValues().contains("-3.14"));
+ Assert.assertTrue("test POJO with both constructor styles supports old calls", doc.getField("summary")
+ .getValues().contains("1025.0"));
+ }
}
diff --git a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedEcho.java b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedEcho.java
new file mode 100644
index 000000000..0c1e63a58
--- /dev/null
+++ b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedEcho.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer.arbitrary;
+
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.Inlinks;
+import org.apache.hadoop.io.Text;
+
+import java.io.PrintStream;
+
+public class UpdatedEcho {
+
+ private NutchDocument doc;
+ private Parse parse;
+ private Text url;
+ private CrawlDatum datum;
+ private Inlinks inlinks;
+ private static PrintStream out = System.out;
+ private String words;
+
+ public UpdatedEcho(String args[],
+ NutchDocument docIn,
+ Parse parseIn,
+ Text urlIn,
+ CrawlDatum datumIn,
+ Inlinks inlinksIn) {
+ super();
+ doc = docIn;
+ parse = parseIn;
+ url = urlIn;
+ inlinks = inlinksIn;
+ datum = datumIn;
+ }
+
+ public UpdatedEcho(String args[]) {
+ super();
+ words = String.valueOf(args[1]);
+ }
+
+ public String getText() {
+ return words;
+ }
+}
diff --git a/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedMultiplier.java b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedMultiplier.java
new file mode 100644
index 000000000..165d69351
--- /dev/null
+++ b/src/plugin/index-arbitrary/src/test/org/apache/nutch/indexer/arbitrary/UpdatedMultiplier.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.indexer.arbitrary;
+
+import org.apache.nutch.parse.Parse;
+import org.apache.nutch.indexer.NutchDocument;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.Inlinks;
+import org.apache.hadoop.io.Text;
+
+import java.io.PrintStream;
+
+public class UpdatedMultiplier {
+ private float product = 1;
+ private static PrintStream err = System.err;
+ private static PrintStream out = System.out;
+
+
+ public UpdatedMultiplier(String args[]) {
+ super();
+ }
+
+ public UpdatedMultiplier(String args[],
+ NutchDocument doc,
+ Parse parse,
+ Text url,
+ CrawlDatum datum,
+ Inlinks inlinks) {
+ super();
+ }
+
+ public String getProduct(String args[]) {
+ int i = args.length - 1;
+ try {
+ while (i >= 0) {
+ product = product * Float.parseFloat(args[i]);
+ i--;
+ }
+ } catch (NumberFormatException nfe) {
+ err.println("NumberFormatException while trying to parse " + String.valueOf(args[i]));
+ }
+ return String.valueOf(product);
+ }
+}