Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

Merge branch 'master' of https://github.com/yacy/yacy_search_server.git

Michael Peter Christen53b01dbf2e6a
diff --cc source/net/yacy/document/importer/ZimImporter.java
index 000000000,2b1f5551c..a1917f0bc
mode 000000,100644..100644
--- a/source/net/yacy/document/importer/ZimImporter.java
+++ b/source/net/yacy/document/importer/ZimImporter.java
@@@ -1,0 -1,306 +1,306 @@@
+ /**
+ * ZimImporter.java
+ * (C) 2023 by Michael Peter Christen @orbiter
+ *
+ * This is a part of YaCy, a peer-to-peer based web search engine
+ *
+ * LICENSE
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+ package net.yacy.document.importer;
+
+ import java.io.File;
+ import java.io.IOException;
+ import java.util.Collection;
+ import java.util.Map;
+ import java.util.TreeMap;
+
+ import net.yacy.cora.document.id.DigestURL;
+ import net.yacy.cora.protocol.RequestHeader;
+ import net.yacy.cora.protocol.ResponseHeader;
+ import net.yacy.cora.util.ConcurrentLog;
+ import net.yacy.crawler.retrieval.Request;
+ import net.yacy.crawler.retrieval.Response;
+ import net.yacy.document.TextParser;
+ import net.yacy.search.Switchboard;
+
+ import org.openzim.ZIMFile;
+ import org.openzim.ZIMReader;
+ import org.openzim.ZIMReader.ArticleEntry;
+ import org.openzim.ZIMReader.DirectoryEntry;
+
+ /**
+ * ZIM importer
+ * can import ZIM file i.e. from https://download.kiwix.org/zim/ or mirrors like https://ftp.fau.de/kiwix/zim/
+ * These files contains identifiers named "URL" which are not actually full URLs but just paths inside a well-known domains.
+ * These domains are sometimes given by a "Source" metadata field, but that is rare - we have to guess them.
+ * For that we have a guessing function, but we must check if the guessing was correct by testing some of the given
+ * URLs against the actual internet-hosted document. Only if that check succeeds we should import the files.
+ * In all other cases the import should work as well but should also only be done in a non-p2p environment to prevent
+ * that such links are shared.
+ */
+ public class ZimImporter extends Thread implements Importer {
+
+ static public ZimImporter job;
+
+ private ZIMFile file;
+ private ZIMReader reader;
+ private String path;
+ private String guessedSource;
+
+ private int recordCnt;
+ private long startTime;
+ private final long sourceSize;
+ private long consumed;
+ private boolean abort = false;
+
+ public ZimImporter(String path) throws IOException {
+ super("ZimImporter - from file " + path);
+ this.path = path;
+ this.file = new ZIMFile(this.path); // this will read already some of the metadata and could consume some time
+ this.sourceSize = this.file.length();
+ }
+
+ @Override
+ public void run() {
+ job = this;
+ this.startTime = System.currentTimeMillis();
+ try {
+ this.reader = new ZIMReader(this.file);
+ this.guessedSource = getSource(this.reader);
+
+ for (int i = 0; i < this.file.header_entryCount; i++) {
+ if (this.abort) break;
+ DirectoryEntry de = this.reader.getDirectoryInfo(i);
+ if (!(de instanceof ZIMReader.ArticleEntry)) continue;
+ ArticleEntry ae = (ArticleEntry) de;
+
+ // check url
+ String guessedUrl = guessURL(this.guessedSource, de);
+ assert guessedUrl.startsWith("http");
+
+ // check availability of text parser
- String mimeType = this.file.getMimeType(ae.mimetype);
++ String mimeType = ae.getMimeType();
+ if (TextParser.supportsMime(mimeType) != null) continue;
+
+ // read the content
+ byte[] b = this.reader.getArticleData(ae);
+
+ // create artificial request and response headers for the indexer
+ RequestHeader requestHeader = new RequestHeader();
+ ResponseHeader responseHeader = new ResponseHeader(200);
+ final Request request = new Request(new DigestURL(guessedUrl), null);
+ final Response response = new Response(
+ request,
+ requestHeader,
+ responseHeader,
+ Switchboard.getSwitchboard().crawler.defaultSurrogateProfile,
+ false,
+ b
+ );
+
+ // throw this to the indexer
+ String error = Switchboard.getSwitchboard().toIndexer(response);
+ if (error != null) ConcurrentLog.info("ZimImporter", "error parsing: " + error);
+ this.recordCnt++;
+ }
+ } catch (IOException e) {
+ ConcurrentLog.info("ZimImporter", "error reading: " + e.getMessage());
+ }
+ ConcurrentLog.info("ZimImporter", "Indexed " + this.recordCnt + " documents");
+ job = null;
+ }
+
+ public void quit() {
+ this.abort = true;
+ }
+
+ @Override
+ public String source() {
+ return this.path;
+ }
+
+ @Override
+ public int count() {
+ return this.recordCnt;
+ }
+
+ @Override
+ public int speed() {
+ if (this.recordCnt == 0) return 0;
+ return (int) (this.recordCnt / Math.max(0L, runningTime() ));
+ }
+
+ @Override
+ public long runningTime() {
+ return (System.currentTimeMillis() - this.startTime) / 1000L;
+ }
+
+ @Override
+ public long remainingTime() {
+ if (this.consumed == 0) {
+ return 0;
+ }
+ long speed = this.consumed / runningTime();
+ return (this.sourceSize - this.consumed) / speed;
+ }
+
+ @Override
+ public String status() {
+ return "";
+ }
+
+ public static String guessDomainName(String fileName) {
+ if (fileName == null || fileName.isEmpty()) {
+ return null; // Handle null or empty input
+ }
+
+ String[] parts = fileName.split("_");
+ if (parts.length == 0) {
+ return null;
+ }
+ String firstPart = parts[0];
+
+ // Handling special cases where the domain name might not be obvious
+ // These are based on your provided list and can be expanded as needed
+ switch (firstPart) {
+ case "100r-off-the-grid":
+ return "100resilientcities.org";
+ case "armypubs":
+ return "armypubs.army.mil";
+ case "artofproblemsolving":
+ return "artofproblemsolving.com";
+ case "based":
+ return "based.cooking";
+ case "booksdash":
+ return "booksdash.com";
+ case "coopmaths":
+ return "coopmaths.fr";
+ case "fas-military-medicine":
+ return "fas.org";
+ case "fonts":
+ return "fonts.google.com";
+ case "gutenberg":
+ return "gutenberg.org";
+ case "ifixit":
+ return "ifixit.com";
+ case "lesfondamentaux":
+ return "reseau-canope.fr";
+ case "lowtechmagazine":
+ return "lowtechmagazine.com";
+ case "mutopiaproject":
+ return "mutopiaproject.org";
+ case "openstreetmap-wiki":
+ return "wiki.openstreetmap.org";
+ case "opentextbooks":
+ return "opentextbooks.org";
+ case "phet":
+ return "phet.colorado.edu";
+ case "practical_action":
+ return "practicalaction.org";
+ case "rapsberry_pi_docs":
+ return "raspberrypi.org";
+ case "ted":
+ return "ted.com";
+ case "vikidia":
+ return "vikidia.org";
+ case "westeros":
+ return "westeros.org";
+ case "wikipedia":
+ return parts[1] + ".wikipedia.org/wiki";
+ case "www.ready.gov":
+ return "ready.gov";
+ }
+
+ // Handling domain patterns
+ if (firstPart.contains(".stackexchange.com")) {
+ return firstPart;
+ } else if (firstPart.endsWith(".com") || firstPart.endsWith(".org") || firstPart.endsWith(".de") ||
+ firstPart.endsWith(".fr") || firstPart.endsWith(".pt") || firstPart.endsWith(".it") ||
+ firstPart.endsWith(".ja") || firstPart.endsWith(".es") || firstPart.endsWith(".eo")) {
+ return firstPart;
+ } else if (firstPart.contains("-")) {
+ return firstPart.substring(0, firstPart.indexOf("-"));
+ }
+
+ // Additional general domain extraction logic
+ if (firstPart.contains(".")) {
+ int lastDotIndex = firstPart.lastIndexOf('.');
+ if (lastDotIndex > 0 && lastDotIndex < firstPart.length() - 1) {
+ // Extract up to the next character beyond the TLD, to support TLDs of variable length
+ int endIndex = firstPart.indexOf('.', lastDotIndex + 1);
+ if (endIndex == -1) {
+ endIndex = firstPart.length();
+ }
+ return firstPart.substring(0, endIndex);
+ }
+ }
+
+ // Default return if none of the above conditions meet
+ return null;
+ }
+
+ public static String getSource(ZIMReader r) throws IOException {
+ String source = r.getMetadata("Source");
+ if (source != null) return source;
+ source = "https://" + guessDomainName(r.getZIMFile().getName()) + "/";
+ return source;
+ }
+
+ public static String guessURL(String guessedSource, DirectoryEntry de) {
+ String url = de.url;
+ if (url.equals("Main_Page")) url = "";
+ return guessedSource + url;
+ }
+
+ public static void main(String[] args) {
+ // zim file import test
+ // will test mostly if domain names are included in zim file urls
+ String zimFilesPath = args[0];
+ File zimFiles = new File(zimFilesPath);
+
+ // make ordered file list; order by file size (start with smallest)
+ String[] filelist = zimFiles.list();
+ Map<Long, File> orderedFileMap = new TreeMap<>();
+ for (int i = 0; i < filelist.length; i++) {
+ if (!filelist[i].endsWith(".zim")) continue;
+ File f = new File(zimFiles, filelist[i]);
+ orderedFileMap.put(f.length() * 1000 + i, f);
+ }
+
+ Collection<File> orderedFiles = orderedFileMap.values();
+ for (File f: orderedFiles) {
+ try {
+ ZIMFile z = new ZIMFile(f.getAbsolutePath());
+ ZIMReader r = new ZIMReader(z);
+ DirectoryEntry de = r.getMainDirectoryEntry();
+ System.out.println("ZIM file: " + f.getAbsolutePath());
+ for (String key: ZIMReader.METADATA_KEYS) {String s = r.getMetadata(key); if (s != null) System.out.println("Metadata " + key + ": " + s);};
+ System.out.println("Namespace: " + de.namespace);
+ System.out.println("Title: " + de.title);
+ System.out.println("URL: " + de.url);
+ System.out.println("guessed domain: " + guessDomainName(f.getName()));
+ String source = getSource(r);
+ System.out.println("guessed Source: " + source);
+ System.out.println("guessed main article: " + guessURL(source, de));
+ System.out.println();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
diff --cc source/org/openzim/ZIMReader.java
index 82a86b479,70dde16bc..9a1f77c5d
--- a/source/org/openzim/ZIMReader.java
+++ b/source/org/openzim/ZIMReader.java
@@@ -19,12 -19,8 +19,13 @@@
package org.openzim;
import java.io.IOException;
--import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
++import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
- import java.util.Map;
++import java.util.Map;
++import java.nio.charset.StandardCharsets;
import org.tukaani.xz.SingleXZInputStream;
import com.github.luben.zstd.ZstdInputStream;
@@@ -45,30 -40,33 +46,35 @@@
*/
public class ZIMReader {
+ public final static String[] METADATA_KEYS = new String[] {
+ "Name", "Title", "Creator", "Publisher", "Date", "Description", "LongDescription",
+ "Language", "License", "Tags", "Relation", "Flavour", "Source", "Counter", "Scraper"
+ };
+
private final ZIMFile mFile;
- public static abstract class DirectoryEntry {
+ public class DirectoryEntry {
- public final int mimetype;
+ private final int mimetype;
public final char namespace;
- public final int cluster_number;
public final String url;
public final String title;
- public final long urlListindex;
+ public final int urlListindex;
-- public DirectoryEntry(
- final long urlListindex,
- final int mimeType, final char namespace,
- final int cluster_number,
- final String url, final String title,
- final int index) {
++ public DirectoryEntry(
++ final int urlListindex,
+ final char namespace, final String url, final String title, final int mimeType) {
+ assert url != null;
- assert title != null;
++ assert title != null;
this.mimetype = mimeType;
this.namespace = namespace;
- this.cluster_number = cluster_number;
this.url = url;
this.title = title;
- this.urlListindex = index;
+ this.urlListindex = urlListindex;
+ }
+
+ public String getMimeType() {
+ return mFile.getMimeType(this.mimetype);
}
}
@@@ -78,26 -76,26 +84,26 @@@
public final int cluster_number;
public final int blob_number;
-- public ArticleEntry(
- final long urlListindex,
- final int mimeType, final char namespace,
- final int cluster_number, final int blob_number,
- final String url, final String title,
- final int urlListindex) {
- super(mimeType, namespace, cluster_number, url, title, urlListindex);
++ public ArticleEntry(
++ final int urlListindex,
+ final char namespace, final String url, final String title, final int mimeType,
+ final int cluster_number, final int blob_number) {
- super(urlListindex, namespace, url, title, mimeType);
++ super(urlListindex, namespace, url, title, mimeType);
this.cluster_number = cluster_number;
this.blob_number = blob_number;
}
}
- public static class RedirectEntry extends DirectoryEntry {
+ public class RedirectEntry extends DirectoryEntry {
- public final long redirect_index;
-
+ public final int redirect_index;
-
- public RedirectEntry(final int mimeType, final char namespace,
- final int redirect_index, final String url, final String title,
- final int urlListindex) {
- super(mimeType, namespace, 0, url, title, urlListindex);
++
+ public RedirectEntry(
- final long urlListindex,
++ final int urlListindex,
+ final char namespace, final String url, final String title, final int mimeType,
- final long redirect_index) {
- super(urlListindex, namespace, url, title, mimeType);
++ final int redirect_index) {
++ super(urlListindex, namespace, url, title, mimeType);
this.redirect_index = redirect_index;
}
@@@ -124,80 -108,25 +130,99 @@@
public ZIMFile getZIMFile() {
return this.mFile;
}
-
++
+ public List<ArticleEntry> getAllArticles() throws IOException {
+ List<ArticleEntry> list = new ArrayList<>();
+ for (int i = 0; i < this.mFile.header_entryCount; i++) {
+ DirectoryEntry de = getDirectoryInfo(i);
+ if (de instanceof ArticleEntry) list.add((ArticleEntry) de);
+ }
+ return list;
+ }
+
+ public Map<Integer, Map<Integer, ArticleEntry>> getIndexedArticles(List<ArticleEntry> list) {
+ Map<Integer, Map<Integer, ArticleEntry>> index = new HashMap<>();
+ for (ArticleEntry entry: list) {
+ Map<Integer, ArticleEntry> cluster = index.get(entry.cluster_number);
+ if (cluster == null) {
+ cluster = new HashMap<Integer, ArticleEntry>();
+ index.put(entry.cluster_number, cluster);
+ }
+ cluster.put(entry.blob_number, entry);
+ }
+ return index;
+ }
+
+ public class ClusterIterator implements Iterator<ArticleBlobEntry> {
+
+ private Map<Integer, Map<Integer, ArticleEntry>> index;
+ private Cluster cluster;
+ private int clusterCounter;
+ private int blobCounter;
+
+ public ClusterIterator() throws IOException {
+ List<ArticleEntry> list = getAllArticles();
+ this.index = getIndexedArticles(list);
+ this.clusterCounter = 0;
+ this.blobCounter = 0;
+ this.cluster = null; // not loaded
+ }
+
+ private final void loadCluster() {
+ if (this.cluster == null) {
+ // load cluster
+ try {
+ this.cluster = new Cluster(this.clusterCounter);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (this.clusterCounter >= mFile.header_clusterCount) return false;
+ loadCluster(); // ensure cluster is loaded
+ return this.blobCounter < this.cluster.blobs.size();
+ }
+
+ @Override
+ public ArticleBlobEntry next() {
+ Map<Integer, ArticleEntry> clusterMap = this.index.get(this.clusterCounter);
+ ArticleEntry ae = clusterMap.get(this.blobCounter);
+ loadCluster(); // ensure cluster is loaded
+ ArticleBlobEntry abe = new ArticleBlobEntry(ae, this.cluster.blobs.get(this.blobCounter));
+
+ // increase the counter(s)
+ this.blobCounter++;
+ if (this.blobCounter >= this.cluster.blobs.size()) {
+ this.clusterCounter++;
+ this.cluster = null; // unload cluster
+ this.blobCounter = 0;
+ }
+ return abe;
+ }
+ }
++
+ public final String getMetadata(String key) throws IOException {
+ DirectoryEntry de = getDirectoryInfo('M', key);
+ if (de == null) return null; // metadata not found; that would be normal
+ byte[] val = getArticleData(de);
+ if (val == null) return null; // article data not found: that is not normal
+ if (val.length == 0) return null; // that empty string is a proper value, however, not usable for a client
+ return new String(val, StandardCharsets.UTF_8);
+ }
+
+ public DirectoryEntry getMainDirectoryEntry() throws IOException {
+ DirectoryEntry de = getDirectoryInfo(this.mFile.header_mainPage);
+ if (de.namespace == 'W' && de.url.equals("mainPage") && de instanceof RedirectEntry) {
+ // resolve redirect to get the actual main page
+ int redirect = ((RedirectEntry) de).redirect_index;
+ de = getDirectoryInfo(redirect);
+ }
- return de;
++ return de;
+ }
public String getURLByURLOrder(final int entryNumber) throws IOException {