Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
added an optimized zim file entry iterator
Michael Peter Christen41856e9f349f
diff --git a/source/org/openzim/ZIMFile.java b/source/org/openzim/ZIMFile.javaindex 2dcfb2208..32ce15539 100644--- a/source/org/openzim/ZIMFile.java+++ b/source/org/openzim/ZIMFile.java@@ -46,10 +46,10 @@ public class ZIMFile extends File {public final int header_majorVersion;public final int header_minorVersion;public final int header_entryCount;- private final int header_clusterCount;+ public final int header_clusterCount;public final long header_urlPtrPos;public final long header_titlePtrPos;- private final long header_clusterPtrPos;+ public final long header_clusterPtrPos;public final long header_mimeListPos;public final int header_mainPage;public final int header_layoutPage;diff --git a/source/org/openzim/ZIMReader.java b/source/org/openzim/ZIMReader.javaindex 427b53072..82a86b479 100644--- a/source/org/openzim/ZIMReader.java+++ b/source/org/openzim/ZIMReader.java@@ -20,6 +20,11 @@ package org.openzim;import java.io.IOException;import java.io.InputStream;+import java.util.ArrayList;+import java.util.HashMap;+import java.util.Iterator;+import java.util.List;+import java.util.Map;import org.tukaani.xz.SingleXZInputStream;import com.github.luben.zstd.ZstdInputStream;@@ -36,65 +41,82 @@ import com.github.luben.zstd.ZstdInputStream;* bugfix to long parsing (prevented reading of large files),* added extended cluster size parsing* added ZStandard compression parsing (cluster type 5)+ * added cluster index*/public class ZIMReader {private final ZIMFile mFile;- public static abstract class DirectoryEntry {+ public class DirectoryEntry {- public final int mimetype;+ private final int mimetype;public final char namespace;- public final int cluster_number;public final String url;public final String title;public final long urlListindex;public DirectoryEntry(- final int mimeType, final char namespace,- final int cluster_number,- final String url, final String title,- final long index) {+ final long urlListindex,+ final char namespace, final String url, final String title, final int mimeType) {+ assert url != null;+ assert title != null;this.mimetype = mimeType;this.namespace = namespace;- this.cluster_number = cluster_number;this.url = url;this.title = title;- this.urlListindex = index;+ this.urlListindex = urlListindex;+ }++ public String getMimeType() {+ return mFile.getMimeType(this.mimetype);}}- public static class ArticleEntry extends DirectoryEntry {+ public class ArticleEntry extends DirectoryEntry {public final int cluster_number;public final int blob_number;public ArticleEntry(- final int mimeType, final char namespace,- final int cluster_number, final int blob_number,- final String url, final String title,- final long urlListindex) {- super(mimeType, namespace, cluster_number, url, title, urlListindex);+ final long urlListindex,+ final char namespace, final String url, final String title, final int mimeType,+ final int cluster_number, final int blob_number) {+ super(urlListindex, namespace, url, title, mimeType);this.cluster_number = cluster_number;this.blob_number = blob_number;}}- public static class RedirectEntry extends DirectoryEntry {+ public class RedirectEntry extends DirectoryEntry {public final long redirect_index;- public RedirectEntry(final int mimeType, final char namespace,- final long redirect_index, final String url, final String title,- final long urlListindex) {- super(mimeType, namespace, 0, url, title, urlListindex);+ public RedirectEntry(+ final long urlListindex,+ final char namespace, final String url, final String title, final int mimeType,+ final long redirect_index) {+ super(urlListindex, namespace, url, title, mimeType);this.redirect_index = redirect_index;}}+ public class ArticleBlobEntry {++ public final ArticleEntry article;+ public final byte[] blob;++ public ArticleBlobEntry(final ArticleEntry article, final byte[] blob) {+ assert article != null;+ assert blob != null;+ this.article = article;+ this.blob = blob;+ }++ }+public ZIMReader(final ZIMFile file) {this.mFile = file;}@@ -103,12 +125,84 @@ public class ZIMReader {return this.mFile;}+ public List<ArticleEntry> getAllArticles() throws IOException {+ List<ArticleEntry> list = new ArrayList<>();+ for (int i = 0; i < this.mFile.header_entryCount; i++) {+ DirectoryEntry de = getDirectoryInfo(i);+ if (de instanceof ArticleEntry) list.add((ArticleEntry) de);+ }+ return list;+ }++ public Map<Integer, Map<Integer, ArticleEntry>> getIndexedArticles(List<ArticleEntry> list) {+ Map<Integer, Map<Integer, ArticleEntry>> index = new HashMap<>();+ for (ArticleEntry entry: list) {+ Map<Integer, ArticleEntry> cluster = index.get(entry.cluster_number);+ if (cluster == null) {+ cluster = new HashMap<Integer, ArticleEntry>();+ index.put(entry.cluster_number, cluster);+ }+ cluster.put(entry.blob_number, entry);+ }+ return index;+ }++ public class ClusterIterator implements Iterator<ArticleBlobEntry> {++ private Map<Integer, Map<Integer, ArticleEntry>> index;+ private Cluster cluster;+ private int clusterCounter;+ private int blobCounter;++ public ClusterIterator() throws IOException {+ List<ArticleEntry> list = getAllArticles();+ this.index = getIndexedArticles(list);+ this.clusterCounter = 0;+ this.blobCounter = 0;+ this.cluster = null; // not loaded+ }++ private final void loadCluster() {+ if (this.cluster == null) {+ // load cluster+ try {+ this.cluster = new Cluster(this.clusterCounter);+ } catch (IOException e) {+ e.printStackTrace();+ }+ }+ }++ @Override+ public boolean hasNext() {+ if (this.clusterCounter >= mFile.header_clusterCount) return false;+ loadCluster(); // ensure cluster is loaded+ return this.blobCounter < this.cluster.blobs.size();+ }++ @Override+ public ArticleBlobEntry next() {+ Map<Integer, ArticleEntry> clusterMap = this.index.get(this.clusterCounter);+ ArticleEntry ae = clusterMap.get(this.blobCounter);+ loadCluster(); // ensure cluster is loaded+ ArticleBlobEntry abe = new ArticleBlobEntry(ae, this.cluster.blobs.get(this.blobCounter));++ // increase the counter(s)+ this.blobCounter++;+ if (this.blobCounter >= this.cluster.blobs.size()) {+ this.clusterCounter++;+ this.cluster = null; // unload cluster+ this.blobCounter = 0;+ }++ return abe;+ }+ }+public String getURLByURLOrder(final int entryNumber) throws IOException {// The position of URL ilong pos = this.mFile.getURLPtr(entryNumber);-- // Move to the position of URL ithis.mFile.mReader.seek(pos);// Article or Redirect entry?@@ -127,19 +221,7 @@ public class ZIMReader {// The articleNumber of the position of URL iint articleNumber = this.mFile.getTitlePtr(entryNumber);- long pos = this.mFile.getURLPtr(articleNumber);- this.mFile.mReader.seek(pos);-- // Article or Redirect entry?- int mimeType = this.mFile.mReader.readTwoLittleEndianBytesInt();-- if (mimeType == 65535) {- this.mFile.mReader.seek(pos + 12);- return this.mFile.mReader.readZeroTerminatedString();- } else {- this.mFile.mReader.seek(pos + 16);- return this.mFile.mReader.readZeroTerminatedString();- }+ return getURLByURLOrder(articleNumber);}public DirectoryEntry getDirectoryInfo(final int entryNumber) throws IOException {@@ -165,14 +247,14 @@ public class ZIMReader {final String url = this.mFile.mReader.readZeroTerminatedString();String title = this.mFile.mReader.readZeroTerminatedString();title = title.equals("") ? url : title;- return new RedirectEntry(type, namespace, redirectIndex, url, title, entryNumber);+ return new RedirectEntry(entryNumber, namespace, url, title, type, redirectIndex);} else {final int cluster_number = this.mFile.mReader.readFourLittleEndianBytesInt(); // 4final int blob_number = this.mFile.mReader.readFourLittleEndianBytesInt(); // 4final String url = this.mFile.mReader.readZeroTerminatedString(); // zero terminatedString title = this.mFile.mReader.readZeroTerminatedString(); // zero terminatedtitle = title.equals("") ? url : title;- return new ArticleEntry(type, namespace, cluster_number, blob_number, url, title, entryNumber);+ return new ArticleEntry(entryNumber, namespace, url, title, type, cluster_number, blob_number);}}@@ -206,6 +288,66 @@ public class ZIMReader {return null;}+ /**+ * Cluster class is required to read a whole cluster with all documents inside at once.+ * This is a good thing because reading single documents from a cluster requires that the+ * cluster is decompressed every time again and again. Doing whole clusters with all documents+ * at once means that the decompression is much more efficient because it is done only once.+ * This can of course only be done, if:+ * - we want to iterate through all documents of a ZIM file+ * - we have reverse indexed all directory entries to be able to assign metadata to cluster documents+ */+ private class Cluster {++ private List<byte[]> blobs;+ private boolean extended;++ public Cluster(int cluster_number) throws IOException {++ // open the cluster and make a Input Stream with the proper decompression type+ final long clusterPos = mFile.geClusterPtr(cluster_number);+ mFile.mReader.seek(clusterPos);+ final int compressionType = mFile.mReader.read();+ InputStream is = null;+ if (compressionType <= 1 || compressionType == 8 || compressionType == 9) {+ extended = compressionType > 1;+ is = mFile.mReader;+ }+ if (compressionType == 4 || compressionType == 12) {+ extended = compressionType == 12;+ is = new SingleXZInputStream(mFile.mReader, 41943040);+ }++ if (compressionType == 5 || compressionType == 13) {+ extended = compressionType == 13;+ is = new ZstdInputStream(mFile.mReader);+ }+ if (is == null) throw new IOException("compression type unknown: " + compressionType);++ // read the offset list+ List<Long> offsets = new ArrayList<>();+ byte[] buffer = new byte[extended ? 8 : 4];+ is.read(buffer);+ long end_offset = extended ? RandomAccessFileZIMInputStream.toEightLittleEndianLong(buffer) : RandomAccessFileZIMInputStream.toFourLittleEndianInteger(buffer);+ offsets.add(end_offset);+ int offset_count = (int) ((end_offset - 1) / (extended ? 8 : 4));+ for (int i = 0; i < offset_count - 1; i++) {+ long l = extended ? RandomAccessFileZIMInputStream.toEightLittleEndianLong(buffer) : RandomAccessFileZIMInputStream.toFourLittleEndianInteger(buffer);+ offsets.add(l);+ }++ // now all document sizes are known because they are defined by the offset deltas+ // the seek position should be now at the beginning of the first document+ this.blobs = new ArrayList<>();+ for (int i = 0; i < offsets.size() - 1; i++) { // loop until the size - 1 because the last offset is the end of the last document+ int length = (int) (offsets.get(i + 1) + offsets.get(i)); // yes the maximum document length is 2GB, for now+ byte[] b = new byte[length];+ RandomAccessFileZIMInputStream.readFully(is, b);+ this.blobs.add(b);+ }+ }+ }+public byte[] getArticleData(final DirectoryEntry directoryInfo) throws IOException {// fail fastdiff --git a/source/org/openzim/ZIMTest.java b/source/org/openzim/ZIMTest.javaindex cb8a28499..8d9c24d4f 100644--- a/source/org/openzim/ZIMTest.java+++ b/source/org/openzim/ZIMTest.java@@ -20,6 +20,7 @@ package org.openzim;import java.io.IOException;import java.nio.charset.StandardCharsets;+import java.util.Iterator;import org.openzim.ZIMReader.DirectoryEntry;@@ -54,6 +55,13 @@ public class ZIMTest {byte[] articleBytes = zReader.getArticleData(directory_entry);String article = articleBytes == null ? "NULL" : new String(articleBytes, StandardCharsets.UTF_8);System.out.println(article);++ // iterate over all entries+ Iterator<ZIMReader.ArticleBlobEntry> i = zReader.new ClusterIterator();+ while (i.hasNext()) {+ ZIMReader.ArticleBlobEntry entry = i.next();+ System.out.println(entry.article.url);+ }} catch (final IOException e) {e.printStackTrace();}