Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

25.4 KB
Raw Blame
/**
 * ZimImporter.java
 * (C) 2023 by Michael Peter Christen @orbiter
 *
 * This is a part of YaCy, a peer-to-peer based web search engine
 *
 * LICENSE
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.
 * If not, see <http://www.gnu.org/licenses/>.
 */

package net.yacy.document.importer;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;

import org.apache.commons.io.FileUtils;
import org.openzim.ZIMFile;
import org.openzim.ZIMReader;
import org.openzim.ZIMReader.ArticleEntry;
import org.openzim.ZIMReader.DirectoryEntry;

import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.crawler.data.CrawlProfile;
import net.yacy.crawler.retrieval.Request;
import net.yacy.crawler.retrieval.Response;
import net.yacy.document.TextParser;
import net.yacy.search.Switchboard;

/**
 * ZIM importer
 * can import ZIM file i.e. from https://download.kiwix.org/zim/ or mirrors like https://ftp.fau.de/kiwix/zim/
 * A huge list is at https://wiki.kiwix.org/wiki/Content_in_all_languages
 * These files contains identifiers named "URL" which are not actually full URLs but just paths inside a well-known domains.
 * These domains are sometimes given by a "Source" metadata field, but that is rare - we have to guess them.
 * For that we have a guessing function, but we must check if the guessing was correct by testing some of the given
 * URLs against the actual internet-hosted document. Only if that check succeeds we should import the files.
 * In all other cases the import should work as well but should also only be done in a non-p2p environment to prevent
 * that such links are shared.
 */
public class ZimImporter extends Thread implements Importer {

    static public ZimImporter job;

    private final ZIMFile file;
    private ZIMReader reader;
    private final String path;
    private String guessedSource;
    private final String collection;

    private int recordCnt;
    private long startTime;
    private final long sourceSize;
    private long consumed;
    private boolean abort = false;

    public ZimImporter(String path, String collection) throws IOException {
       super("ZimImporter - from file " + path);
       this.path = path;
       this.file = new ZIMFile(this.path); // this will read already some of the metadata and could consume some time
       this.sourceSize = this.file.length();
       this.collection = collection;
    }

    public ZimImporter(String path, InputStream is, String collection) throws IOException {
        super("ZimImporter - from file " + path);
        this.path = path;
        File zimFilePath = new File(path);
        if (!zimFilePath.exists() && is != null) {
            File tempFile = File.createTempFile(zimFilePath.getName().substring(0, zimFilePath.getName().length() - 4), "zim");
            FileUtils.copyInputStreamToFile(is, tempFile);
            tempFile.deleteOnExit();
            this.file = new ZIMFile(tempFile.getPath());
        } else {
            this.file = new ZIMFile(this.path); // this will read already some of the metadata and could consume some time
        }
        this.sourceSize = this.file.length();
        this.collection = collection;
    }

    @Override
    public void run() {
        job = this;
        this.startTime = System.currentTimeMillis();
        final Switchboard sb = Switchboard.getSwitchboard();
        try {
            this.reader = new ZIMReader(this.file);
            this.guessedSource = getSource(this.reader, this.file.getName());
            final Date guessedDate = getDate(this.reader);
            final String dates = HeaderFramework.newRfc1123Format().format(guessedDate);

            // verify the source
            final DirectoryEntry mainEntry = this.reader.getMainDirectoryEntry();
            final DigestURL mainURL = guessURL(this.guessedSource, mainEntry);
            //if (!mainURL.exists(ClientIdentification.browserAgent)) {
            //    sb.log.info("zim importer: file " + this.file.getName() + " failed main url existence test: " + mainURL);
            //    return;
            //}

            final CrawlProfile zimProfile = (CrawlProfile) Switchboard.getSwitchboard().crawler.defaultPackProfile.clone();
            zimProfile.setCollections(this.collection);
            zimProfile.setHandle();

            // read all documents
            for (int i = 0; i < this.file.header_entryCount; i++) {
                try {
                    if (this.abort) break;
                    final DirectoryEntry de = this.reader.getDirectoryInfo(i);
                    if (!(de instanceof ZIMReader.ArticleEntry)) continue;
                    final ArticleEntry ae = (ArticleEntry) de;
                    if (ae.namespace != 'C' && ae.namespace != 'A') continue;

                    // check url
                    final DigestURL guessedUrl = guessURL(this.guessedSource, de);
                    //if (this.recordCnt < 10) {
                        // critical test for the first 10 urls
                    //    if (!guessedUrl.exists(ClientIdentification.browserAgent)) {
                    //        sb.log.info("zim importer: file " + this.file.getName() + " failed url " + this.recordCnt + " existence test: " + guessedUrl);
                    //        return;
                    //    }
                    //}

                    // check availability of text parser
                    final String mimeType = ae.getMimeType();
                    if (!mimeType.startsWith("text/") && !mimeType.equals("application/epub+zip")) continue; // in this import we want only text, not everything that is possible
                    if (TextParser.supportsMime(mimeType) != null) continue;

                    // read the content
                    final byte[] b = this.reader.getArticleData(ae);

                    // create artificial request and response headers for the indexer
                    final RequestHeader requestHeader = new RequestHeader();
                    final ResponseHeader responseHeader = new ResponseHeader(200);
                    responseHeader.put(HeaderFramework.CONTENT_TYPE, de.getMimeType()); // very important to tell parser which kind of content
                    responseHeader.put(HeaderFramework.LAST_MODIFIED, dates); // put in the guessd date to have something that is not the current date
                    final Request request = new Request(
                            ASCII.getBytes(sb.peers.mySeed().hash),
                            guessedUrl,
                            null, // referrerhash the hash of the referrer URL
                            de.title, // name the name of the document to crawl
                            null, // appdate the time when the url was first time appeared
                            zimProfile.handle(),        // profileHandle the name of the prefetch profile. This must not be null!
                            0,    // depth the crawling depth of the entry
                            zimProfile.timezoneOffset() // timezone offset
                    );
                    final Response response = new Response(
                            request,
                            requestHeader,
                            responseHeader,
                            zimProfile,
                            false,
                            b
                    );

                    // throw this to the indexer
                    final String error = sb.toIndexer(response);
                    if (error != null) ConcurrentLog.info("ZimImporter", "error parsing: " + error);
                    this.recordCnt++;
                } catch (final Exception e) {
                    // catch any error that could stop the importer
                    ConcurrentLog.info("ZimImporter", "error loading: " + e.getMessage());
                }
            }
        } catch (final IOException e) {
            ConcurrentLog.info("ZimImporter", "error reading: " + e.getMessage());
        }
        ConcurrentLog.info("ZimImporter", "Indexed " + this.recordCnt + " documents");
        job = null;
    }

    public void quit() {
        this.abort = true;
    }

    @Override
    public String source() {
        return this.path;
    }

    @Override
    public int count() {
        return this.recordCnt;
    }

    @Override
    public int speed() {
        if (this.recordCnt == 0) return 0;
        return (int) (this.recordCnt / Math.max(0L, this.runningTime() ));
    }

    @Override
    public long runningTime() {
        return (System.currentTimeMillis() - this.startTime) / 1000L;
    }

    @Override
    public long remainingTime() {
        if (this.consumed == 0) {
            return 0;
        }
        final long speed = this.consumed / this.runningTime();
        return (this.sourceSize - this.consumed) / speed;
    }

    @Override
    public String status() {
        return "";
    }

    public static String guessDomainName(String fileName) {
        if (fileName == null || fileName.isEmpty()) {
            return null; // Handle null or empty input
        }

        final String[] parts = fileName.split("_");
        if (parts.length == 0) {
            return null;
        }
        final String firstPart = parts[0];

        // Handling special cases where the domain name might not be obvious
        // These are based on your provided list and can be expanded as needed
        switch (firstPart) {
            case "100r-off-the-grid":
                return "100resilientcities.org";
            case "armypubs":
                return "armypubs.army.mil";
            case "artofproblemsolving":
                return "artofproblemsolving.com";
            case "based":
                return "based.cooking";
            case "booksdash":
                return "booksdash.com";
            case "coopmaths":
                return "coopmaths.fr";
            case "fas-military-medicine":
                return "fas.org";
            case "fonts":
                return "fonts.google.com";
            case "ifixit":
                return "ifixit.com";
            case "lesfondamentaux":
                return "reseau-canope.fr";
            case "lowtechmagazine":
                return "lowtechmagazine.com";
            case "mutopiaproject":
                return "mutopiaproject.org";
            case "openstreetmap-wiki":
                return "wiki.openstreetmap.org";
            case "opentextbooks":
                return "opentextbooks.org";
            case "phet":
                return "phet.colorado.edu";
            case "practical_action":
                return "practicalaction.org";
            case "rapsberry_pi_docs":
                return "raspberrypi.org";
            case "ted":
                return "www.ted.com/search?q=";
            case "vikidia":
                return parts[1] + ".vikidia.org/wiki";
            case "westeros":
                return "westeros.org";
            case "mdwiki":
                return "mdwiki.org/wiki";
            case "wikihow":
                return parts[1].equals("en") ? "wikihow.com" : parts[1] + ".wikihow.com";
            case "wikisource":
                return parts[1] + ".wikisource.org/wiki";
            case "wikiversity":
                return parts[1] + ".wikiversity.org/wiki";
            case "wikivoyage":
                return parts[1] + ".wikivoyage.org/wiki";
            case "wiktionary":
                return parts[1] + ".wiktionary.org/wiki";
            case "wikiquote":
                return parts[1] + ".wikiquote.org/wiki";
            case "wikibooks":
                return parts[1] + ".wikibooks.org/wiki";
            case "wikinews":
                return parts[1] + ".wikinews.org/wiki";
            case "wikipedia":
                return parts[1] + ".wikipedia.org/wiki";
            case "www.ready.gov":
                return "ready.gov";
        }

        // Handling domain patterns
        if (firstPart.contains(".stackexchange.com")) {
            return firstPart;
        } else if (firstPart.endsWith(".com") || firstPart.endsWith(".org") || firstPart.endsWith(".de") ||
                   firstPart.endsWith(".fr") || firstPart.endsWith(".pt") || firstPart.endsWith(".it") ||
                   firstPart.endsWith(".ja") || firstPart.endsWith(".es") || firstPart.endsWith(".eo")) {
            return firstPart;
        } else if (firstPart.contains("-")) {
            return firstPart.substring(0, firstPart.indexOf("-"));
        }

        // Additional general domain extraction logic
        if (firstPart.contains(".")) {
            final int lastDotIndex = firstPart.lastIndexOf('.');
            if (lastDotIndex > 0 && lastDotIndex < firstPart.length() - 1) {
                // Extract up to the next character beyond the TLD, to support TLDs of variable length
                int endIndex = firstPart.indexOf('.', lastDotIndex + 1);
                if (endIndex == -1) {
                    endIndex = firstPart.length();
                }
                return firstPart.substring(0, endIndex);
            }
        }

        // Default return if none of the above conditions meet
        return null;
    }

    public static String getSource(ZIMReader r, String fileName) throws IOException {
        String source = r.getMetadata("Source");
        if (source != null) return source;
        if (fileName == null || fileName.length() == 0) fileName = r.getZIMFile().getName();
        source = "https://" + guessDomainName(fileName) + "/";
        return source;
    }

    public static Date getDate(ZIMReader r) throws IOException {
        final String date = r.getMetadata("Date");
        if (date != null) try {
            final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd", Locale.US);
            return format.parse(date);
        } catch (final ParseException e) {}
        // failover situation: use file date
        return new Date(r.getZIMFile().lastModified());
    }

    public static DigestURL guessURL(String guessedSource, DirectoryEntry de) throws MalformedURLException {
        String url = de.url;
        if (url.equals("Main_Page")) url = "";
        if (url.startsWith("A/")) return new DigestURL("https://" + url.substring(2));
        if (url.startsWith("H/")) return new DigestURL("https://" + url.substring(2));
        if (guessedSource != null) return new DigestURL(guessedSource + url);
        return new DigestURL(guessedSource + url);
    }

    private final static String[] skip_files = {
         "iota.stackexchange.com_en_all_2023-05.zim",
         "stellar.stackexchange.com_en_all_2023-10.zim",
         "vegetarianism.stackexchange.com_en_all_2023-05.zim",
         "esperanto.stackexchange.com_eo_all_2023-10.zim",
         "tezos.stackexchange.com_en_all_2023-10.zim",
         "eosio.stackexchange.com_en_all_2023-10.zim",
         "ebooks.stackexchange.com_en_all_2023-10.zim",
         "poker.stackexchange.com_en_all_2023-05.zim",
         "cseducators.stackexchange.com_en_all_2023-10.zim",
         "iot.stackexchange.com_en_all_2023-05.zim",
         "portuguese.stackexchange.com_pt_all_2023-04.zim",
         "portuguese.stackexchange.com_pt_all_2023-10.zim",
         "italian.stackexchange.com_it_all_2023-05.zim",
         "monero.stackexchange.com_en_all_2022-11.zim",
         "sustainability.stackexchange.com_en_all_2023-05.zim",
         "westeros_en_all_nopic_2021-03.zim",
         "opensource.stackexchange.com_en_all_2023-10.zim",
         "tor.stackexchange.com_en_all_2023-05.zim",
         "devops.stackexchange.com_en_all_2023-10.zim",
         "patents.stackexchange.com_en_all_2023-10.zim",
         "stackapps.com_en_all_2023-05.zim",
         "hardwarerecs.stackexchange.com_en_all_2023-05.zim",
         "hsm.stackexchange.com_en_all_2023-05.zim",
         "expatriates.stackexchange.com_en_all_2023-11.zim",
         "opendata.stackexchange.com_en_all_2023-10.zim",
         "sports.stackexchange.com_en_all_2023-05.zim",
         "wikinews_de_all_nopic_2023-10.zim",
         "computergraphics.stackexchange.com_en_all_2023-10.zim",
         "tridion.stackexchange.com_en_all_2023-10.zim",
         "bioinformatics.stackexchange.com_en_all_2023-10.zim",
         "expressionengine.stackexchange.com_en_all_2023-11.zim",
         "elementaryos.stackexchange.com_en_all_2023-10.zim",
         "cstheory.stackexchange.com_en_all_2023-10.zim",
         "chess.stackexchange.com_en_all_2023-05.zim",
         "vi.stackexchange.com_en_all_2023-05.zim",
         "fitness.stackexchange.com_en_all_2023-10.zim",
         "pets.stackexchange.com_en_all_2023-05.zim",
         "french.stackexchange.com_fr_all_2023-10.zim",
         "sqa.stackexchange.com_en_all_2023-05.zim",
         "islam.stackexchange.com_en_all_2023-05.zim",
         "scicomp.stackexchange.com_en_all_2023-05.zim",
         "wikinews_en_all_nopic_2023-09.zim",
         "ai.stackexchange.com_en_all_2023-10.zim",
         "boardgames.stackexchange.com_en_all_2023-05.zim",
         "economics.stackexchange.com_en_all_2023-05.zim",
         "3dprinting.stackexchange.com_en_all_2023-07.zim",
         "earthscience.stackexchange.com_en_all_2023-05.zim",
         "emacs.stackexchange.com_en_all_2023-10.zim",
         "bitcoin.stackexchange.com_en_all_2023-05.zim",
         "philosophy.stackexchange.com_en_all_2023-05.zim",
         "law.stackexchange.com_en_all_2023-05.zim",
         "astronomy.stackexchange.com_en_all_2023-05.zim",
         "artofproblemsolving_en_all_nopic_2021-03.zim",
         "engineering.stackexchange.com_en_all_2023-05.zim",
         "ja.stackoverflow.com_ja_all_2023-06.zim",
         "webmasters.stackexchange.com_en_all_2023-05.zim",
         "anime.stackexchange.com_en_all_2023-10.zim",
         "cooking.stackexchange.com_en_all_2023-05.zim",
         "arduino.stackexchange.com_en_all_2023-05.zim",
         "money.stackexchange.com_en_all_2023-05.zim",
         "judaism.stackexchange.com_en_all_2023-05.zim",
         "ethereum.stackexchange.com_en_all_2023-05.zim",
         "datascience.stackexchange.com_en_all_2023-10.zim",
         "academia.stackexchange.com_en_all_2023-10.zim",
         "music.stackexchange.com_en_all_2023-05.zim",
         "cs.stackexchange.com_en_all_2023-03.zim",
         "dsp.stackexchange.com_en_all_2023-05.zim",
         "biology.stackexchange.com_en_all_2023-05.zim",
         "android.stackexchange.com_en_all_2023-10.zim",
         "bicycles.stackexchange.com_en_all_2023-05.zim",
         "puzzling.stackexchange.com_en_all_2023-05.zim",
         "photo.stackexchange.com_en_all_2023-05.zim",
         "aviation.stackexchange.com_en_all_2023-05.zim",
         "drupal.stackexchange.com_en_all_2023-05.zim",
         "ux.stackexchange.com_en_all_2023-05.zim",
         "ell.stackexchange.com_en_all_2023-10.zim",
         "openstreetmap-wiki_en_all_nopic_2023-05.zim",
         "softwareengineering.stackexchange.com_en_all_2023-05.zim",
         "gaming.stackexchange.com_en_all_2023-10.zim",
         "mathematica.stackexchange.com_en_all_2023-10.zim",
         "pt.stackoverflow.com_pt_all_2023-06.zim",
         "apple.stackexchange.com_en_all_2023-05.zim",
         "diy.stackexchange.com_en_all_2023-08.zim",
         "es.stackoverflow.com_es_all_2023-06.zim",
         "gis.stackexchange.com_en_all_2023-05.zim",
         "stats.stackexchange.com_en_all_2023-05.zim",
         "physics.stackexchange.com_en_all_2023-05.zim",
         "serverfault.com_en_all_2023-05.zim",
         "electronics.stackexchange.com_en_all_2023-05.zim",
         "tex.stackexchange.com_en_all_2023-05.zim",
         "wikibooks_de_all_nopic_2021-03.zim",
         "askubuntu.com_en_all_2023-05.zim",
         "superuser.com_en_all_2023-05.zim",
         "lesfondamentaux.reseau-canope.fr_fr_all_2022-11.zim",
         "wikibooks_en_all_nopic_2021-03.zim",
         "courses.lumenlearning.com_en_all_2021-03.zim",
         "wikipedia_de_all_nopic_2023-10.zim",
         "wikipedia_en_all_nopic_2023-10.zim",
         "stackoverflow.com_en_all_nopic_2022-07.zim",
         "stackoverflow.com_en_all_2023-05.zim",
         "armypubs_en_all_2023-08.zim",
         "vikidia_en_all_nopic_2023-09.zim",
         "wikiquote_de_all_nopic_2023-10.zim",
         "wikiquote_en_all_nopic_2023-09.zim",
         "wiktionary_de_all_nopic_2023-10.zim",
         "wiktionary_en_all_nopic_2023-10.zim",
         "wikihow_de_maxi_2023-10.zim",
         "wikivoyage_de_all_nopic_2023-09.zim",
         "wikiversity_de_all_nopic_2021-03.zim",
         "wikiversity_en_all_nopic_2021-03.zim",
         "wikisource_de_all_nopic_2023-09.zim",
         "wikisource_en_all_nopic_2023-08.zim",
         "ted_countdown_global_2023-09.zim",
         "ted_en_design_2023-09.zim",
         "ted_en_business_2023-09.zim",
         "ted_en_global_issues_2023-09.zim",
         "opentextbooks_en_all_2023-08.zim",
         "bestedlessons.org_en_all_2023-08.zim",
         "wikivoyage_en_all_nopic_2023-10.zim",
         "based.cooking_en_all_2023-10.zim",
         "wordnet_en_all_2023-04.zim",
         "internet-encyclopedia-philosophy_en_all_2023-08.zim",
         "100r-off-the-grid_en_2023-09.zim",
         "coopmaths_2023-04.zim",
         "birds-of-ladakh_en_all_2023-02.zim",
         "storyweaver.org_en_2023-09.zim",
         "developer.mozilla.org_en_all_2023-02.zim",
         "www.ready.gov_es_2023-06.zim",
         "teoria.com_en_2023-08.zim",
         "theworldfactbook_en_all_2023-06.zim",
         "mutopiaproject.org_en_2023-08.zim",
         "dp.la_en_all_2023-08.zim",

         // 302
         "moderators.stackexchange.com_en_all_2023-05.zim",
         "beer.stackexchange.com_en_all_2023-05.zim",
         "health.stackexchange.com_en_all_2023-05.zim",
         "avp.stackexchange.com_en_all_2023-05.zim",
         "lowtechmagazine.com_en_all_2023-08.zim",
         "ifixit_de_all_2023-07.zim",
         "ifixit_en_all_2023-10.zim",
         "der-postillon.com_de_all_2020-12.zim",
         "wikihow_en_maxi_2023-03.zim",
    };

    public static void main(String[] args) {
        final Set<String> skip = new HashSet<>();
        for (final String s: skip_files) skip.add(s);
        // zim file import test
        // will test mostly if domain names are included in zim file urls
        final String zimFilesPath = args[0];
        final File zimFiles = new File(zimFilesPath);

        // make ordered file list; order by file size (start with smallest)
        final String[] filelist = zimFiles.list();
        final Map<Long, File> orderedFileMap = new TreeMap<>();
        for (int i = 0; i < filelist.length; i++) {
            if (!filelist[i].endsWith(".zim")) continue;
            final File f = new File(zimFiles, filelist[i]);
            orderedFileMap.put(f.length() * 1000 + i, f);
        }

        final Collection<File> orderedFiles = orderedFileMap.values();
        final Set<String> files_ok = new LinkedHashSet<>();
        final Set<String> files_nok = new LinkedHashSet<>();
        for (final File f: orderedFiles) {
            if (skip.contains(f.getName())) continue;
            try {
                final ZIMFile z = new ZIMFile(f.getAbsolutePath());
                final ZIMReader r = new ZIMReader(z);
                final DirectoryEntry de = r.getMainDirectoryEntry();
                System.out.println("ZIM file:  " + f.getAbsolutePath());
                for (final String key: ZIMReader.METADATA_KEYS) {final String s = r.getMetadata(key); if (s != null) System.out.println("Metadata " + key + ": " + s);}
                System.out.println("Namespace: " + de.namespace);
                System.out.println("Title:     " + de.title);
                System.out.println("URL:       " + de.url);
                System.out.println("Mime Type  " + de.getMimeType());
                System.out.println("guessed domain: " + guessDomainName(f.getName())); // uses a table and rules that deduces a source from the file name
                final String source = getSource(r, null);
                System.out.println("guessed Source: " + source); // this uses metadata stored in the zim file
                final DigestURL mainURL = guessURL(source, de);
                System.out.println("guessed main article: " + mainURL);
                final boolean ok = mainURL.exists(ClientIdentification.browserAgent);
                System.out.println("main article exists: " + ok);
                if (ok) files_ok.add(f.getName()); else files_nok.add(f.getName());
                System.out.println();
            } catch (final IOException e) {
                e.printStackTrace();
            }
        }
        System.out.println("ok files: " + files_ok.toString());
        System.out.println("not-ok files: " + files_nok.toString());
    }
}