Apache Nutch is an extensible and scalable web crawler
NUTCH-3085 Augment CI by adding code coverage and code quality reporting (#897)
Lewis John McGibbney39cfc6169647
diff --git a/.github/workflows/master-build.yml b/.github/workflows/master-build.ymlindex 2056c1e10..76d98db4e 100644--- a/.github/workflows/master-build.yml+++ b/.github/workflows/master-build.yml@@ -246,3 +246,11 @@ jobs:./build/test/TEST-*.xml./build/**/test/TEST-*.xmlretention-days: 1+ - name: Upload Coverage Data+ uses: actions/upload-artifact@v4+ if: always() && matrix.os == 'ubuntu-latest'+ with:+ name: coverage-data+ path: ./build/coverage/*.exec+ retention-days: 1+ if-no-files-found: ignorediff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.ymlnew file mode 100644index 000000000..e591c11c5--- /dev/null+++ b/.github/workflows/sonarcloud.yml@@ -0,0 +1,67 @@+# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++name: sonarcloud+on:+ workflow_run:+ workflows: [master pull request ci]+ types: [completed]+jobs:+ analysis:+ if: github.event.workflow_run.conclusion == 'success'+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v5+ with:+ repository: ${{ github.event.workflow_run.head_repository.full_name }}+ ref: ${{ github.event.workflow_run.head_sha }}+ fetch-depth: 0+ - name: Set up JDK 17+ uses: actions/setup-java@v5+ with:+ java-version: '17'+ distribution: 'temurin'+ - name: Cache Ivy dependencies+ uses: actions/cache@v4+ with:+ path: ~/.ivy2/cache+ key: ${{ runner.os }}-ivy-${{ hashFiles('ivy/ivy.xml', 'src/plugin/**/ivy.xml') }}+ restore-keys: |+ ${{ runner.os }}-ivy-+ - name: Compile (no tests)+ run: ant compile compile-plugins -buildfile build.xml+ - name: Download coverage data+ uses: dawidd6/action-download-artifact@v11+ with:+ name: coverage-data+ workflow: master-build.yml+ run_id: ${{ github.event.workflow_run.id }}+ path: ./build/coverage/+ continue-on-error: true+ - name: Download test reports+ uses: dawidd6/action-download-artifact@v11+ with:+ name: junit-test-results-ubuntu-latest+ workflow: master-build.yml+ run_id: ${{ github.event.workflow_run.id }}+ path: ./build/test/+ continue-on-error: true+ - name: Generate JaCoCo XML report+ run: ant jacoco-report -buildfile build.xml+ continue-on-error: true+ - name: SonarCloud Scan+ uses: SonarSource/sonarcloud-github-action@master+ env:+ SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}diff --git a/README.md b/README.mdindex f1322aa5e..30fdc1d5a 100644--- a/README.md+++ b/README.md@@ -2,6 +2,7 @@ Apache Nutch README===================[](https://github.com/apache/nutch/actions/workflows/master-build.yml)+[](https://sonarcloud.io/summary/new_code?id=apache_nutch)<img src="https://nutch.apache.org/assets/img/nutch_logo_tm.png" align="right" width="300" />diff --git a/build.xml b/build.xmlindex 57d44ee49..796f4bfc7 100644--- a/build.xml+++ b/build.xml@@ -44,6 +44,11 @@<property name="spotbugs.home" value="${ivy.dir}/spotbugs-${spotbugs.version}" /><property name="spotbugs.jar" value="${spotbugs.home}/lib/spotbugs-ant.jar" />+ <property name="jacoco.home" value="${ivy.dir}/jacoco-${jacoco.version}" />+ <property name="jacoco.agent.jar" value="${jacoco.home}/lib/jacocoagent.jar" />+ <property name="jacoco.ant.jar" value="${jacoco.home}/lib/jacocoant.jar" />+ <property name="coverage.dir" value="${build.dir}/coverage" />+<property name="apache-rat.version" value="0.16" /><property name="apache-rat.home" value="${ivy.dir}/apache-rat-${apache-rat.version}" /><property name="apache-rat.jar" value="${apache-rat.home}/apache-rat-${apache-rat.version}.jar" />@@ -490,9 +495,10 @@</antcall></target>- <target name="test-core" depends="compile-core-test, job" description="--> run core JUnit tests">+ <target name="test-core" depends="compile-core-test, job, jacoco-download" description="--> run core JUnit tests"><delete dir="${test.build.data}"/><mkdir dir="${test.build.data}"/>+ <mkdir dir="${coverage.dir}"/><copy todir="${test.build.data}"><fileset dir="src/testresources" includes="**/*"/></copy>@@ -507,6 +513,7 @@<listener type="legacy-xml" sendSysOut="true" sendSysErr="true"/><fork forkMode="perTestClass"><jvmarg value="-Xmx1000m"/>+ <jvmarg value="-javaagent:${jacoco.agent.jar}=destfile=${coverage.dir}/jacoco-core.exec,append=true"/><sysproperty key="test.build.data" value="${test.build.data}"/><sysproperty key="test.src.dir" value="${test.src.dir}"/><sysproperty key="test.include.slow" value="${test.include.slow}"/>@@ -523,6 +530,7 @@<listener type="legacy-xml" sendSysOut="true" sendSysErr="true"/><fork forkMode="perTestClass"><jvmarg value="-Xmx1000m"/>+ <jvmarg value="-javaagent:${jacoco.agent.jar}=destfile=${coverage.dir}/jacoco-core.exec,append=true"/><sysproperty key="test.build.data" value="${test.build.data}"/><sysproperty key="test.src.dir" value="${test.src.dir}"/><sysproperty key="test.include.slow" value="${test.include.slow}"/>@@ -537,7 +545,7 @@<fail if="tests.failed">Tests failed!</fail></target>- <target name="test-plugins" depends="resolve-test, compile, compile-core-test" description="--> run JUnit tests for all plugins">+ <target name="test-plugins" depends="resolve-test, compile, compile-core-test, jacoco-download" description="--> run JUnit tests for all plugins"><ant dir="src/plugin" target="test" inheritAll="false"/></target>@@ -1125,6 +1133,58 @@</spotbugs></target>+ <!-- ================================================================== -->+ <!-- JaCoCo code coverage -->+ <!-- ================================================================== -->+ <target name="jacoco-download" description="--> download JaCoCo">+ <available file="${jacoco.agent.jar}" property="jacoco.jar.found"/>+ <antcall target="jacoco-download-unchecked"/>+ </target>++ <target name="jacoco-download-unchecked" unless="jacoco.jar.found"+ description="--> downloads the JaCoCo distribution zip.">+ <get src="https://github.com/jacoco/jacoco/releases/download/v${jacoco.version}/jacoco-${jacoco.version}.zip"+ dest="${ivy.dir}/jacoco-${jacoco.version}.zip" usetimestamp="false" />++ <mkdir dir="${jacoco.home}"/>+ <unzip src="${ivy.dir}/jacoco-${jacoco.version}.zip" dest="${jacoco.home}" />++ <delete file="${ivy.dir}/jacoco-${jacoco.version}.zip" />+ </target>++ <target name="jacoco-report" depends="jacoco-download" description="--> generate JaCoCo XML coverage report">+ <taskdef uri="antlib:org.jacoco.ant" resource="org/jacoco/ant/antlib.xml">+ <classpath>+ <pathelement location="${jacoco.ant.jar}" />+ </classpath>+ </taskdef>++ <jacoco:merge destfile="${coverage.dir}/jacoco-merged.exec" xmlns:jacoco="antlib:org.jacoco.ant">+ <fileset dir="${coverage.dir}" includes="*.exec"/>+ </jacoco:merge>++ <jacoco:report xmlns:jacoco="antlib:org.jacoco.ant">+ <executiondata>+ <file file="${coverage.dir}/jacoco-merged.exec"/>+ </executiondata>+ <structure name="Apache Nutch">+ <classfiles>+ <fileset dir="${build.classes}"/>+ <fileset dir="${build.plugins}">+ <include name="**/*.jar"/>+ </fileset>+ </classfiles>+ <sourcefiles encoding="${build.encoding}">+ <fileset dir="${src.dir}"/>+ <fileset dir="${plugins.dir}">+ <include name="*/src/java/**/*.java"/>+ </fileset>+ </sourcefiles>+ </structure>+ <xml destfile="${coverage.dir}/jacoco.xml"/>+ </jacoco:report>+ </target>+<!-- ================================================================== --><!-- Eclipse targets --><!-- ================================================================== -->diff --git a/default.properties b/default.propertiesindex 68a9b304d..e0fde46d8 100644--- a/default.properties+++ b/default.properties@@ -43,6 +43,9 @@ test.build.javadoc = ${test.build.dir}/docs/api# Enable with: ant test -Dtest.failfast=truetest.failfast = false+# JaCoCo code coverage+jacoco.version=0.8.12+# Proxy Host and Port to use for building JavaDocjavadoc.proxy.host=-J-DproxyHost=javadoc.proxy.port=-J-DproxyPort=diff --git a/sonar-project.properties b/sonar-project.propertiesnew file mode 100644index 000000000..a128b8f24--- /dev/null+++ b/sonar-project.properties@@ -0,0 +1,31 @@+# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++sonar.projectKey=apache_nutch+sonar.organization=apache+sonar.projectName=Apache Nutch++sonar.sources=src/java,src/plugin+sonar.tests=src/test,src/plugin+sonar.test.inclusions=**/src/test/**/*.java,**/Test*.java+sonar.source.encoding=UTF-8+sonar.java.source=11++sonar.java.binaries=build/classes,build/plugins+sonar.java.test.binaries=build/test/classes+sonar.java.libraries=build/lib/*.jar++sonar.coverage.jacoco.xmlReportPaths=build/coverage/jacoco.xml+sonar.junit.reportPaths=build/testdiff --git a/src/plugin/build-plugin.xml b/src/plugin/build-plugin.xmlindex ef8dda56c..7b07810ae 100755--- a/src/plugin/build-plugin.xml+++ b/src/plugin/build-plugin.xml@@ -44,6 +44,9 @@<!-- load nutch defaults last so that they can be overridden above --><property file="${nutch.root}/default.properties" />+ <property name="jacoco.agent.jar" location="${nutch.root}/ivy/jacoco-${jacoco.version}/lib/jacocoagent.jar"/>+ <property name="coverage.dir" location="${nutch.root}/build/coverage"/>+<ivy:settings id="ivy.instance" file="${nutch.root}/ivy/ivysettings.xml" /><path id="plugin.deps"/>@@ -209,6 +212,7 @@<!-- ================================================================== --><target name="test" depends="compile-test, deploy" if="test.available"><echo message="Testing plugin: ${name}"/>+ <mkdir dir="${coverage.dir}"/><junitlauncher printSummary="true" haltOnFailure="true" failureProperty="tests.failed"><classpath refid="test.classpath"/><testclasses outputDir="${build.test}" unless="testcase">@@ -216,6 +220,7 @@<listener type="legacy-xml" sendSysOut="true" sendSysErr="true"/><fork forkMode="perTestClass"><jvmarg value="-Xmx1000m"/>+ <jvmarg value="-javaagent:${jacoco.agent.jar}=destfile=${coverage.dir}/jacoco-plugin-${name}.exec,append=true"/><sysproperty key="test.data" value="${build.test}/data"/><sysproperty key="test.input" value="${root}/data"/><sysproperty key="junit.platform.execution.failfast.enabled" value="${test.failfast}"/>@@ -231,6 +236,7 @@<listener type="legacy-xml" sendSysOut="true" sendSysErr="true"/><fork forkMode="perTestClass"><jvmarg value="-Xmx1000m"/>+ <jvmarg value="-javaagent:${jacoco.agent.jar}=destfile=${coverage.dir}/jacoco-plugin-${name}.exec,append=true"/><sysproperty key="test.data" value="${build.test}/data"/><sysproperty key="test.input" value="${root}/data"/><sysproperty key="junit.platform.execution.failfast.enabled" value="${test.failfast}"/>