Skip to content

armgit / nutch

Public Imported from GitHub
0

Apache Nutch is an extensible and scalable web crawler

13.1 KB
Raw Blame
<?xml version="1.0"?>
<!--
 Licensed to the Apache Software Foundation (ASF) under one or more
 contributor license agreements.  See the NOTICE file distributed with
 this work for additional information regarding copyright ownership.
 The ASF licenses this file to You under the Apache License, Version 2.0
 (the "License"); you may not use this file except in compliance with
 the License.  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-->
<project name="Nutch" default="deploy-core" basedir=".">

  <target name="deploy-core">
    <ant target="compile-core" inheritall="false" dir="../.."/>
    <ant target="deploy"/>
  </target>

  <!-- ====================================================== -->
  <!-- Build & deploy all the plugin jars.                    -->
  <!-- ====================================================== -->
  <target name="deploy">
    <!-- first build libs, other plugins depend on them -->
    <ant dir="lib-htmlunit" target="deploy"/>
    <ant dir="lib-http" target="deploy"/>
    <ant dir="lib-nekohtml" target="deploy"/>
    <ant dir="lib-rabbitmq" target="deploy"/>
    <ant dir="lib-regex-filter" target="deploy"/>
    <ant dir="lib-selenium" target="deploy"/>
    <ant dir="lib-xml" target="deploy"/>
    <!-- real plugins -->
    <ant dir="creativecommons" target="deploy"/>
    <ant dir="feed" target="deploy"/>
    <ant dir="headings" target="deploy"/>
    <ant dir="exchange-jexl" target="deploy"/>
    <ant dir="index-anchor" target="deploy"/>
    <ant dir="index-arbitrary" target="deploy"/>
    <ant dir="index-basic" target="deploy"/>
    <ant dir="index-geoip" target="deploy"/>
    <ant dir="index-jexl-filter" target="deploy"/>
    <ant dir="index-links" target="deploy"/>
    <ant dir="index-metadata" target="deploy"/>
    <ant dir="index-more" target="deploy"/>
    <ant dir="index-replace" target="deploy"/>
    <ant dir="index-static" target="deploy"/>
    <ant dir="indexer-cloudsearch" target="deploy"/>
    <ant dir="indexer-csv" target="deploy"/>
    <ant dir="indexer-dummy" target="deploy"/>
    <ant dir="indexer-elastic" target="deploy"/>
    <ant dir="indexer-kafka" target="deploy"/>
    <ant dir="indexer-opensearch-1x" target="deploy"/>
    <ant dir="indexer-rabbit" target="deploy"/>
    <ant dir="indexer-solr" target="deploy"/>
    <ant dir="language-identifier" target="deploy"/>
    <ant dir="microformats-reltag" target="deploy"/>
    <ant dir="mimetype-filter" target="deploy"/>
    <ant dir="nutch-extensionpoints" target="deploy"/>
    <ant dir="parse-ext" target="deploy"/>
    <ant dir="parse-html" target="deploy"/>
    <ant dir="parse-js" target="deploy"/>
    <ant dir="parse-metatags" target="deploy"/>
    <ant dir="parse-tika" target="deploy"/>
    <ant dir="parse-zip" target="deploy"/>
    <ant dir="parsefilter-debug" target="deploy"/>
    <ant dir="parsefilter-naivebayes" target="deploy"/>
    <ant dir="parsefilter-regex" target="deploy"/>
    <ant dir="protocol-file" target="deploy"/>
    <ant dir="protocol-foo" target="deploy" />
    <ant dir="protocol-ftp" target="deploy"/>
    <ant dir="protocol-htmlunit" target="deploy" />
    <ant dir="protocol-http" target="deploy"/>
    <ant dir="protocol-httpclient" target="deploy"/>
    <ant dir="protocol-interactiveselenium" target="deploy" />
    <ant dir="protocol-okhttp" target="deploy"/>
    <ant dir="protocol-selenium" target="deploy" />
    <ant dir="publish-rabbitmq" target="deploy"/>
    <ant dir="scoring-depth" target="deploy"/>
    <ant dir="scoring-link" target="deploy"/>
    <ant dir="scoring-opic" target="deploy"/>
    <ant dir="scoring-orphan" target="deploy"/>
    <ant dir="scoring-similarity" target="deploy"/>
    <ant dir="scoring-metadata"  target="deploy"/>
    <ant dir="subcollection" target="deploy"/>
    <ant dir="tld" target="deploy"/>
    <ant dir="urlfilter-automaton" target="deploy"/>
    <ant dir="urlfilter-domain" target="deploy" />
    <ant dir="urlfilter-domaindenylist" target="deploy" />
    <ant dir="urlfilter-fast" target="deploy"/>
    <ant dir="urlfilter-prefix" target="deploy"/>
    <ant dir="urlfilter-regex" target="deploy"/>
    <ant dir="urlfilter-suffix" target="deploy"/>
    <ant dir="urlfilter-validator" target="deploy"/>
    <!-- urlfilter-ignoreexempt depends on urlfilter-regex, must build after -->
    <ant dir="urlfilter-ignoreexempt" target="deploy"/>
    <ant dir="urlmeta" target="deploy"/>
    <ant dir="urlnormalizer-ajax" target="deploy"/>
    <ant dir="urlnormalizer-basic" target="deploy"/>
    <ant dir="urlnormalizer-host" target="deploy"/>
    <ant dir="urlnormalizer-pass" target="deploy"/>
    <ant dir="urlnormalizer-protocol" target="deploy"/>
    <ant dir="urlnormalizer-querystring" target="deploy"/>
    <ant dir="urlnormalizer-regex" target="deploy"/>
    <ant dir="urlnormalizer-slash" target="deploy"/>
  </target>

  <!-- ====================================================== -->
  <!-- Test all of the plugins.                               -->
  <!-- ====================================================== -->
  <target name="test">
    <parallel threadCount="2">
     <ant dir="creativecommons" target="test"/>
     <ant dir="feed" target="test"/>
     <ant dir="headings" target="test"/>
     <ant dir="index-anchor" target="test"/>
     <ant dir="index-arbitrary" target="test"/>
     <ant dir="index-basic" target="test"/>
     <ant dir="index-geoip" target="test"/>
     <ant dir="index-jexl-filter" target="test"/>
     <ant dir="index-links" target="test"/>
     <ant dir="index-more" target="test"/>
     <ant dir="index-replace" target="test"/>
     <ant dir="index-static" target="test"/>
     <ant dir="indexer-csv" target="test"/>
     <ant dir="language-identifier" target="test"/>
     <ant dir="lib-http" target="test"/>
     <ant dir="lib-regex-filter" target="test"/>
     <ant dir="mimetype-filter" target="test"/>
     <!--ant dir="parse-ext" target="test"/-->
     <ant dir="parse-html" target="test"/>
     <ant dir="parse-js" target="test"/>
     <ant dir="parse-metatags" target="test"/>
     <ant dir="parse-tika" target="test"/>
     <ant dir="parse-zip" target="test"/>
     <ant dir="parsefilter-regex" target="test"/>
     <ant dir="protocol-file" target="test"/>
     <ant dir="protocol-http" target="test"/>
     <ant dir="protocol-httpclient" target="test"/>
     <ant dir="protocol-okhttp" target="test"/>
     <ant dir="scoring-orphan" target="test"/>
     <ant dir="scoring-metadata" target="test"/>
     <ant dir="subcollection" target="test"/>
     <ant dir="urlfilter-automaton" target="test"/>
     <ant dir="urlfilter-domain" target="test"/>
     <ant dir="urlfilter-domaindenylist" target="test"/>
     <ant dir="urlfilter-fast" target="test"/>
     <!--ant dir="urlfilter-ignoreexempt" target="test"/-->
     <ant dir="urlfilter-prefix" target="test"/>
     <ant dir="urlfilter-regex" target="test"/>
     <ant dir="urlfilter-suffix" target="test"/>
     <ant dir="urlfilter-validator" target="test"/>
     <ant dir="urlnormalizer-ajax" target="test"/>
     <ant dir="urlnormalizer-basic" target="test"/>
     <ant dir="urlnormalizer-host" target="test"/>
     <ant dir="urlnormalizer-pass" target="test"/>
     <ant dir="urlnormalizer-protocol" target="test"/>
     <ant dir="urlnormalizer-querystring" target="test"/>
     <ant dir="urlnormalizer-regex" target="test"/>
     <ant dir="urlnormalizer-slash" target="test"/>
    </parallel>
  </target>

  <!-- ====================================================== -->
  <!-- Test a single plugin.                                  -->
  <!-- ====================================================== -->
  <target name="test-single">
    <ant dir="${plugin}" target="test"/>
  </target>

  <!-- ======================================================  -->
  <!-- Indexer plugin integration tests (Testcontainers)       -->
  <!-- Run sequentially to avoid container resource contention -->
  <!-- ======================================================  -->
  <target name="test-indexer-integration">
    <ant dir="indexer-elastic" target="test-indexer-integration"/>
    <ant dir="indexer-kafka" target="test-indexer-integration"/>
    <ant dir="indexer-rabbit" target="test-indexer-integration"/>
    <ant dir="indexer-solr" target="test-indexer-integration"/>
  </target>

  <!-- ======================================================  -->
  <!-- Protocol plugin integration tests (Testcontainers)    -->
  <!-- Run sequentially to avoid container resource contention-->
  <!-- ======================================================  -->
  <target name="test-protocol-integration">
    <ant dir="protocol-ftp"        target="test-protocol-integration"/>
    <ant dir="protocol-http"       target="test-protocol-integration"/>
    <ant dir="protocol-httpclient" target="test-protocol-integration"/>
    <ant dir="protocol-htmlunit"   target="test-protocol-integration"/>
    <ant dir="protocol-okhttp"     target="test-protocol-integration"/>
    <ant dir="protocol-selenium"   target="test-protocol-integration"/>
  </target>

  <!-- ====================================================== -->
  <!-- Clean all of the plugins.                              -->
  <!-- ====================================================== -->
  <target name="clean">
    <ant dir="creativecommons" target="clean"/>
    <ant dir="feed" target="clean"/>
    <ant dir="headings" target="clean"/>
    <ant dir="exchange-jexl" target="clean"/>
    <ant dir="index-anchor" target="clean"/>
    <ant dir="index-arbitrary" target="clean"/>
    <ant dir="index-basic" target="clean"/>
    <ant dir="index-geoip" target="clean"/>
    <ant dir="index-jexl-filter" target="clean"/>
    <ant dir="index-links" target="clean"/>
    <ant dir="index-metadata" target="clean"/>
    <ant dir="index-more" target="clean"/>
    <ant dir="index-replace" target="clean"/>
    <ant dir="index-static" target="clean"/>
    <ant dir="indexer-cloudsearch" target="clean"/>
    <ant dir="indexer-csv" target="clean"/>
    <ant dir="indexer-dummy" target="clean"/>
    <ant dir="indexer-elastic" target="clean"/>
    <ant dir="indexer-kafka" target="clean"/>
    <ant dir="indexer-opensearch-1x" target="clean"/>
    <ant dir="indexer-rabbit" target="clean"/>
    <ant dir="indexer-solr" target="clean"/>
    <ant dir="language-identifier" target="clean"/>
    <ant dir="lib-htmlunit" target="clean"/>
    <ant dir="lib-http" target="clean"/>
    <ant dir="lib-nekohtml" target="clean"/>
    <ant dir="lib-rabbitmq" target="clean"/>
    <ant dir="lib-regex-filter" target="clean"/>
    <ant dir="lib-selenium" target="clean"/>
    <ant dir="lib-xml" target="clean"/>
    <ant dir="microformats-reltag" target="clean"/>
    <ant dir="mimetype-filter" target="clean"/>
    <ant dir="nutch-extensionpoints" target="clean"/>
    <ant dir="parse-ext" target="clean"/>
    <ant dir="parse-html" target="clean"/>
    <ant dir="parse-js" target="clean"/>
    <ant dir="parse-metatags" target="clean"/>
    <ant dir="parse-tika" target="clean"/>
    <ant dir="parse-zip" target="clean"/>
    <ant dir="parsefilter-debug" target="clean" />
    <ant dir="parsefilter-naivebayes" target="clean" />
    <ant dir="parsefilter-regex" target="clean"/>
    <ant dir="protocol-file" target="clean"/>
    <ant dir="protocol-foo" target="clean" />
    <ant dir="protocol-ftp" target="clean"/>
    <ant dir="protocol-htmlunit" target="clean" />
    <ant dir="protocol-http" target="clean"/>
    <ant dir="protocol-httpclient" target="clean"/>
    <ant dir="protocol-interactiveselenium" target="clean" />
    <ant dir="protocol-okhttp" target="clean"/>
    <ant dir="protocol-selenium" target="clean" />
    <ant dir="publish-rabbitmq" target="clean"/>
    <ant dir="scoring-depth" target="clean"/>
    <ant dir="scoring-link" target="clean"/>
    <ant dir="scoring-opic" target="clean"/>
    <ant dir="scoring-orphan" target="clean"/>
    <ant dir="scoring-similarity" target="clean"/>
    <ant dir="scoring-metadata" target="clean"/>
    <ant dir="subcollection" target="clean"/>
    <ant dir="tld" target="clean"/>
    <ant dir="urlfilter-automaton" target="clean"/>
    <ant dir="urlfilter-domain" target="clean" />
    <ant dir="urlfilter-domaindenylist" target="clean" />
    <ant dir="urlfilter-fast" target="clean"/>
    <ant dir="urlfilter-ignoreexempt" target="clean"/>
    <ant dir="urlfilter-prefix" target="clean"/>
    <ant dir="urlfilter-regex" target="clean"/>
    <ant dir="urlfilter-suffix" target="clean"/>
    <ant dir="urlfilter-validator" target="clean"/>
    <ant dir="urlmeta" target="clean"/>
    <ant dir="urlnormalizer-ajax" target="clean"/>
    <ant dir="urlnormalizer-basic" target="clean"/>
    <ant dir="urlnormalizer-host" target="clean"/>
    <ant dir="urlnormalizer-pass" target="clean"/>
    <ant dir="urlnormalizer-protocol" target="clean"/>
    <ant dir="urlnormalizer-querystring" target="clean"/>
    <ant dir="urlnormalizer-regex" target="clean"/>
    <ant dir="urlnormalizer-slash" target="clean"/>
  </target>
</project>