Apache Nutch is an extensible and scalable web crawler
NUTCH-3197 Yetus: fix real precommit defects (shell, Docker, shelldocs) (#958)
Lewis John McGibbneyd062e5e334fa
diff --git a/.hadolint.yaml b/.hadolint.yamlnew file mode 100644index 000000000..c7a9df49c--- /dev/null+++ b/.hadolint.yaml@@ -0,0 +1,4 @@+# Hadolint config for docker/Dockerfile (NUTCH-3197).+# DL3018: do not pin floating Alpine apk versions without a deliberate bump policy.+ignored:+ - DL3018diff --git a/.yetus/excludes.txt b/.yetus/excludes.txtindex 6cfe88d3c..c7d261c44 100644--- a/.yetus/excludes.txt+++ b/.yetus/excludes.txt@@ -14,3 +14,6 @@## Naive Bayes training sample (deliberately messy email/OCR-like text).^conf/naivebayes-train\.txt\.template$+#+# parse-js sample: test fixture for link extraction, not production JS+^src/plugin/parse-js/sample/diff --git a/README.md b/README.mdindex 2a4ec82fe..e2841edc1 100644--- a/README.md+++ b/README.md@@ -48,7 +48,7 @@ test-patch --basedir=/path/to/clean/repo --build-tool=nobuild \```Exclude patterns and related Yetus baselines can be added under `.yetus/`-(see `.yetus/excludes.txt`).+(see `.yetus/excludes.txt`, `.yetus/detsecrets-ignored-hashes.txt`).IDE setup---------diff --git a/src/bin/crawl b/src/bin/crawlindex 409f72799..07ac7789d 100755--- a/src/bin/crawl+++ b/src/bin/crawl@@ -23,7 +23,7 @@## Options:# -i|--index Indexes crawl results into a configured indexer-# -D <propery>=<value> A Nutch or Hadoop property to pass to Nutch calls overwriting+# -D <property>=<value> A Nutch or Hadoop property to pass to Nutch calls overwriting# properties defined in configuration files, e.g.# increase content limit to 2MB:# -D http.content.limit=2097152@@ -51,6 +51,8 @@# --dedup-group <none|host|domain> Deduplication group method [default: none]#+## @audience private+## @stability stablefunction __to_seconds() {NUMBER=$(echo $1 | tr -dc '0-9')MODIFIER=$(echo $1 | tr -dc '[^s|h|m|d]]')@@ -73,6 +75,8 @@ function __to_seconds() {echo $SECONDS}+## @audience private+## @stability stablefunction __print_usage {echo "Usage: crawl [options] <crawl_dir> <num_rounds>"echo -e ""@@ -129,7 +133,7 @@ NUM_THREADS=50SITEMAPS_FROM_HOSTDB_FREQUENCY=neverDEDUP_GROUP=none-while [[ $# > 0 ]]+while [[ $# -gt 0 ]]docase $1 in-i|--index)@@ -229,9 +233,12 @@ bin="`cd "$bin"; pwd`"# determines whether mode based on presence of job filemode=local-if [ -f "${bin}"/../*nutch*.job ]; then- mode=distributed-fi+for f in "${bin}"/../*nutch*.job; do+ if [ -f "$f" ]; then+ mode=distributed+ break+ fi+doneif [[ "$mode" = "local" ]]; thenif [[ "$NUM_FETCHERS" -ne 1 ]]; thenecho "Ignoring configured number of fetchers (--num_fetchers): a single fetcher task is used when running in local mode."@@ -247,28 +254,33 @@ commonOptions=("${HADOOP_PROPERTIES[@]}" -Dmapreduce.job.reduces=$NUM_TASKS -Dmaif [ $mode = "distributed" ]; thenif [ $(which hadoop | wc -l ) -eq 0 ]; thenecho "Can't find Hadoop executable. Add HADOOP_COMMON_HOME/bin to the path or run in local mode."- exit -1;+ # POSIX: 127 = utility not found (hadoop not on PATH)+ exit 127;fifi+## @audience private+## @stability stablefunction __bin_nutch {# run $bin/nutch, exit if exit value indicates error- echo "$bin/nutch $@" ;# echo command and arguments+ echo "$bin/nutch" "$@" ;# echo command and arguments"$bin/nutch" "$@"RETCODE=$?if [ $RETCODE -ne 0 ]thenecho "Error running:"- echo " $bin/nutch $@"+ echo " $bin/nutch" "$@"echo "Failed with exit value $RETCODE."exit $RETCODEfi}# check if directory exists locally or on hdfs+## @audience private+## @stability stablefunction __directory_exists {if [[ "$mode" == local && -d "$1" ]]; thenreturn 0@@ -279,6 +291,8 @@ function __directory_exists {fi}+## @audience private+## @stability stablefunction __update_hostdb {if __directory_exists "$CRAWL_PATH"/crawldb; thenecho "Updating HostDB"@@ -334,7 +348,7 @@ dogenerate_args=("${commonOptions[@]}" "$CRAWL_PATH"/crawldb "$CRAWL_PATH"/segments -topN $SIZE_FETCHLIST -numFetchers $NUM_FETCHERS -noFilter)fi- echo "$bin/nutch generate ${generate_args[@]}"+ echo "$bin/nutch generate" "${generate_args[@]}"$bin/nutch generate "${generate_args[@]}"RETCODE=$?if [ $RETCODE -eq 0 ]; then@@ -352,7 +366,7 @@ dofielseecho "Error running:"- echo " $bin/nutch generate ${generate_args[@]}"+ echo " $bin/nutch generate" "${generate_args[@]}"echo "Failed with exit value $RETCODE."exit $RETCODEfidiff --git a/src/bin/nutch b/src/bin/nutchindex 255a8c43e..00fca5aa4 100755--- a/src/bin/nutch+++ b/src/bin/nutch@@ -147,17 +147,18 @@ filocal=true-# NUTCH_JOB-if [ -f "${NUTCH_HOME}"/*nutch*.job ]; then- local=false- for f in "$NUTCH_HOME"/*nutch*.job; do+# NUTCH_JOB+for f in "${NUTCH_HOME}"/*nutch*.job; do+ if [ -f "$f" ]; then+ local=falseNUTCH_JOB="$f"- done- # cygwin path translation- if $cygwin; then- NUTCH_JOB="`cygpath -p -w "$NUTCH_JOB"`"+ # cygwin path translation+ if $cygwin; then+ NUTCH_JOB="$(cygpath -p -w "$NUTCH_JOB")"+ fi+ breakfi-fi+doneJAVA="$JAVA_HOME/bin/java"JAVA_HEAP_MAX=-Xmx4096m@@ -238,7 +239,7 @@ fi# figure out which class to runif [ "$COMMAND" = "crawl" ] ; thenecho "Command $COMMAND is deprecated, please use bin/crawl instead"- exit -1+ exit 1elif [ "$COMMAND" = "inject" ] ; thenCLASS=org.apache.nutch.crawl.Injectorelif [ "$COMMAND" = "generate" ] ; then@@ -272,7 +273,7 @@ elif [ "$COMMAND" = "commoncrawldump" ] ; thenelif [ "$COMMAND" = "solrindex" ] || [ "$COMMAND" = "solrdedup" ] || [ "$COMMAND" = "solrclean" ]; thenREPLACEMENT="${COMMAND#solr}"echo "The command $COMMAND was replaced by the command $REPLACEMENT"- exit -1+ exit 1elif [ "$COMMAND" = "index" ] ; thenCLASS=org.apache.nutch.indexer.IndexingJobelif [ "$COMMAND" = "dedup" ] ; then@@ -336,7 +337,8 @@ else# check that hadoop can be found on the pathif [ $(which hadoop | wc -l ) -eq 0 ]; thenecho "Can't find Hadoop executable. Add HADOOP_COMMON_HOME/bin to the path or run in local mode."- exit -1;+ # POSIX: 127 = utility not found (hadoop not on PATH)+ exit 127;fifi