Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
removed synchronization in table creation to avoid possible deadlocks when handling OnDemandOpenFileIndex which happens quite often during wide crawling
Michael Peter Christen4cadd557dcdf
diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.javaindex 27d1f2aa1..f2ef7f455 100644--- a/source/net/yacy/kelondro/table/Table.java+++ b/source/net/yacy/kelondro/table/Table.java@@ -36,6 +36,7 @@ import java.util.List;import java.util.Map;import java.util.TreeMap;import java.util.TreeSet;+import java.util.concurrent.ConcurrentSkipListMap;import net.yacy.cora.document.encoding.ASCII;import net.yacy.cora.order.CloneableIterator;@@ -71,9 +72,9 @@ public class Table implements Index, Iterable<Row.Entry> {// static tracker objectsprivate final static ConcurrentLog log = new ConcurrentLog("TABLE");-+/** Map all active table instances by file name */- private final static TreeMap<String, Table> tableTracker = new TreeMap<String, Table>();+ private final static Map<String, Table> tableTracker = new ConcurrentSkipListMap<String, Table>();private final static long maxarraylength = 134217727L; // (2^27-1) that may be the maximum size of array length in some JVMsprivate final long minmemremaining; // if less than this memory is remaininig, the memory copy of a table is abandoned@@ -85,13 +86,13 @@ public class Table implements Index, Iterable<Row.Entry> {private RowSet table;public Table(- final File tablefile,- final Row rowdef,- final int buffersize,- final int initialSpace,- boolean useTailCache,- final boolean exceed134217727,- final boolean warmUp) throws SpaceExceededException, kelondroException {+ final File tablefile,+ final Row rowdef,+ final int buffersize,+ final int initialSpace,+ boolean useTailCache,+ final boolean exceed134217727,+ final boolean warmUp) throws SpaceExceededException, kelondroException {this.rowdef = rowdef;this.buffersize = buffersize;@@ -111,7 +112,7 @@ public class Table implements Index, Iterable<Row.Entry> {freshFile = true;FileOutputStream fos = null;if(tablefile.getParentFile() != null) {- tablefile.getParentFile().mkdirs();+ tablefile.getParentFile().mkdirs();}try {fos = new FileOutputStream(tablefile);@@ -131,17 +132,17 @@ public class Table implements Index, Iterable<Row.Entry> {final int records = Math.max(fileSize, initialSpace);final long neededRAM4table = 200L * 1024L * 1024L + records * (this.taildef.objectsize + rowdef.primaryKeyLength + 4L) * 3L / 2L;this.table = null;-+try {this.table = ((exceed134217727 || neededRAM4table < maxarraylength) &&- useTailCache && MemoryControl.available() > 600L * 1024L * 1024L &&- MemoryControl.request(neededRAM4table, true)) ? new RowSet(this.taildef, records) : null;+ useTailCache && MemoryControl.available() > 600L * 1024L * 1024L &&+ MemoryControl.request(neededRAM4table, true)) ? new RowSet(this.taildef, records) : null;} catch (final SpaceExceededException e) {- this.table = null;+ this.table = null;} catch (final Throwable e) {- this.table = null;+ this.table = null;}-+if (log.isFine()) log.fine("initialization of " + tablefile.getName() + ". table copy: " + ((this.table == null) ? "no" : "yes") + ", available RAM: " + (MemoryControl.available() / 1024L / 1024L) + "MB, needed: " + (neededRAM4table / 1024L / 1024L) + "MB, allocating space for " + records + " entries");final long neededRAM4index = 100L * 1024L * 1024L + records * (rowdef.primaryKeyLength + 4L) * 3L / 2L;if (records > 0 && !MemoryControl.request(neededRAM4index, true)) {@@ -163,75 +164,75 @@ public class Table implements Index, Iterable<Row.Entry> {if (this.table == null) {final ChunkIterator ki = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.primaryKeyLength);try {- while (ki.hasNext()) {- key = ki.next();- // write the key into the index table- assert key != null;- if (key == null) {i++; continue;}- if (rowdef.objectOrder.wellformed(key)) {- this.index.putUnique(key, i++);- } else {- errors.putUnique(key, i++);- }- }+ while (ki.hasNext()) {+ key = ki.next();+ // write the key into the index table+ assert key != null;+ if (key == null) {i++; continue;}+ if (rowdef.objectOrder.wellformed(key)) {+ this.index.putUnique(key, i++);+ } else {+ errors.putUnique(key, i++);+ }+ }} finally {- /* If any error occurred while looping over the iterator, we- * must ensure the underlying stream is closed before- * transmitting the exception to the upper layer- */- if(ki.hasNext()) {- try {- ki.close();- } catch(IOException ioe) {- /* Do not block if closing is not possible but anyway keep a trace in log */- log.warn("Could not close input stream on the file " + tablefile);- }- }+ /* If any error occurred while looping over the iterator, we+ * must ensure the underlying stream is closed before+ * transmitting the exception to the upper layer+ */+ if(ki.hasNext()) {+ try {+ ki.close();+ } catch(IOException ioe) {+ /* Do not block if closing is not possible but anyway keep a trace in log */+ log.warn("Could not close input stream on the file " + tablefile);+ }+ }}} else {byte[] record;key = new byte[rowdef.primaryKeyLength];final ChunkIterator ri = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.objectsize);try {- while (ri.hasNext()) {- record = ri.next();- assert record != null;- if (record == null) {i++; continue;}- System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength);-- // write the key into the index table- if (rowdef.objectOrder.wellformed(key)) {- this.index.putUnique(key, i++);- // write the tail into the table- try {- this.table.addUnique(this.taildef.newEntry(record, rowdef.primaryKeyLength, true));- } catch (final SpaceExceededException e) {- this.table = null;- try {- ri.close(); // close inputstream of chunkiterator- } finally {- /* Do not block if closing is not possible but anyway keep a trace in log */- log.warn("Could not close input stream on the file " + tablefile);- }- break;- }- } else {- errors.putUnique(key, i++);- }- }+ while (ri.hasNext()) {+ record = ri.next();+ assert record != null;+ if (record == null) {i++; continue;}+ System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength);++ // write the key into the index table+ if (rowdef.objectOrder.wellformed(key)) {+ this.index.putUnique(key, i++);+ // write the tail into the table+ try {+ this.table.addUnique(this.taildef.newEntry(record, rowdef.primaryKeyLength, true));+ } catch (final SpaceExceededException e) {+ this.table = null;+ try {+ ri.close(); // close inputstream of chunkiterator+ } finally {+ /* Do not block if closing is not possible but anyway keep a trace in log */+ log.warn("Could not close input stream on the file " + tablefile);+ }+ break;+ }+ } else {+ errors.putUnique(key, i++);+ }+ }} finally {- /* If any error occurred while looping over the iterator, we- * must ensure the underlying stream is closed before- * transmitting the exception to the upper layer- */- if(ri.hasNext()) {- try {- ri.close();- } catch(IOException ioe) {- /* Do not block if closing is not possible but anyway keep a trace in log */- log.warn("Could not close input stream on the file " + tablefile);- }- }+ /* If any error occurred while looping over the iterator, we+ * must ensure the underlying stream is closed before+ * transmitting the exception to the upper layer+ */+ if(ri.hasNext()) {+ try {+ ri.close();+ } catch(IOException ioe) {+ /* Do not block if closing is not possible but anyway keep a trace in log */+ log.warn("Could not close input stream on the file " + tablefile);+ }+ }}Runtime.getRuntime().gc();if (abandonTable()) {@@ -270,14 +271,14 @@ public class Table implements Index, Iterable<Row.Entry> {}// track this table- synchronized (tableTracker) {tableTracker.put(tablefile.toString(), this);}+ tableTracker.put(tablefile.toString(), this);}public synchronized void warmUp() {warmUp0();}- private void warmUp0() {+ private final void warmUp0() {// remove doublestry {final ArrayList<long[]> doubles = this.index.removeDoubles();@@ -378,62 +379,62 @@ public class Table implements Index, Iterable<Row.Entry> {* Table memory usage statistics*/public class TableStatistics {- /** The table entries number */- private int tableSize = 0;-- /** Size of a key chunk in bytes */- private int keyChunkSize = 0;-- /** Total size of keys in bytes */- private long keyMem = 0;-- /** Size of a value chunk in bytes */- private int valueChunkSize = 0;-- /** Total size of values in bytes */- private long valueMem = 0;-- /**- * @return the size of a key chunk in bytes- */- public int getKeyChunkSize() {- return this.keyChunkSize;- }-- /**- * @return the total size of the table in-memory keys in bytes- */- public long getKeyMem() {- return this.keyMem;- }-- /**- * @return the table entries number- */- public int getTableSize() {- return this.tableSize;- }-- /**- * @return the size of a value chunk in bytes- */- public int getValueChunkSize() {- return this.valueChunkSize;- }-- /**- * @return the total size of the table in-memory values in bytes- */- public long getValueMem() {- return this.valueMem;- }-- /**- * @return the total memory used by the table in bytes- */- public long getTotalMem() {- return this.keyMem + this.valueMem;- }+ /** The table entries number */+ private int tableSize = 0;++ /** Size of a key chunk in bytes */+ private int keyChunkSize = 0;++ /** Total size of keys in bytes */+ private long keyMem = 0;++ /** Size of a value chunk in bytes */+ private int valueChunkSize = 0;++ /** Total size of values in bytes */+ private long valueMem = 0;++ /**+ * @return the size of a key chunk in bytes+ */+ public int getKeyChunkSize() {+ return this.keyChunkSize;+ }++ /**+ * @return the total size of the table in-memory keys in bytes+ */+ public long getKeyMem() {+ return this.keyMem;+ }++ /**+ * @return the table entries number+ */+ public int getTableSize() {+ return this.tableSize;+ }++ /**+ * @return the size of a value chunk in bytes+ */+ public int getValueChunkSize() {+ return this.valueChunkSize;+ }++ /**+ * @return the total size of the table in-memory values in bytes+ */+ public long getValueMem() {+ return this.valueMem;+ }++ /**+ * @return the total memory used by the table in bytes+ */+ public long getTotalMem() {+ return this.keyMem + this.valueMem;+ }}/**@@ -448,12 +449,12 @@ public class Table implements Index, Iterable<Row.Entry> {if (this.index == null) return stats; // possibly closed or being closedstats.tableSize = this.index.size();if(this.index instanceof RowHandleMap) {- stats.keyChunkSize = (((RowHandleMap) this.index).row().objectsize);+ stats.keyChunkSize = (((RowHandleMap) this.index).row().objectsize);stats.keyMem = (long)((RowHandleMap) this.index).row().objectsize * (long)this.index.size();}- if(table != null) {- stats.valueChunkSize = this.table.row().objectsize;- stats.valueMem = (long)this.table.row().objectsize * (long)this.table.size();+ if(this.table != null) {+ stats.valueChunkSize = this.table.row().objectsize;+ stats.valueMem = (long)this.table.row().objectsize * (long)this.table.size();}return stats;@@ -579,17 +580,17 @@ public class Table implements Index, Iterable<Row.Entry> {@Overridepublic void close() {- String tablefile = null;+ String tablefile = null;if (this.file != null) {- tablefile = this.file.filename().toString();- this.file.close();+ tablefile = this.file.filename().toString();+ this.file.close();}this.file = null;if (this.table != null) this.table.close();this.table = null;if (this.index != null) this.index.close();this.index = null;- if (tablefile != null) tableTracker.remove(tablefile);+ if (tablefile != null) tableTracker.remove(tablefile);}@Override@@ -616,8 +617,8 @@ public class Table implements Index, Iterable<Row.Entry> {}}- private Entry get0(final byte[] key) throws IOException {- if (this.file == null || this.index == null) return null;+ private final Entry get0(final byte[] key) throws IOException {+ if (this.file == null || this.index == null) return null;final int i = (int) this.index.get(key);if (i == -1) return null;final byte[] b = new byte[this.rowdef.objectsize];@@ -643,7 +644,7 @@ public class Table implements Index, Iterable<Row.Entry> {}@Override- public Map<byte[], Row.Entry> get(final Collection<byte[]> keys, final boolean forcecopy) throws IOException, InterruptedException {+ public final Map<byte[], Row.Entry> get(final Collection<byte[]> keys, final boolean forcecopy) throws IOException, InterruptedException {final Map<byte[], Row.Entry> map = new TreeMap<byte[], Row.Entry>(row().objectOrder);Row.Entry entry;for (final byte[] key: keys) {@@ -654,13 +655,13 @@ public class Table implements Index, Iterable<Row.Entry> {}@Override- public boolean has(final byte[] key) {+ public final boolean has(final byte[] key) {if (this.index == null) return false;return this.index.has(key);}@Override- public synchronized CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException {+ public final synchronized CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException {return this.index.keys(up, firstKey);}@@ -723,7 +724,7 @@ public class Table implements Index, Iterable<Row.Entry> {* @throws SpaceExceededException*/@Override- public boolean put(final Entry row) throws IOException, SpaceExceededException {+ public final boolean put(final Entry row) throws IOException, SpaceExceededException {assert row != null;if (this.file == null || row == null) return true;final byte[] rowb = row.bytes();@@ -769,7 +770,7 @@ public class Table implements Index, Iterable<Row.Entry> {* @throws IOException* @throws SpaceExceededException*/- private void removeInFile(final int i) throws IOException, SpaceExceededException {+ private final void removeInFile(final int i) throws IOException, SpaceExceededException {assert i >= 0;final byte[] p = new byte[this.rowdef.objectsize];@@ -828,12 +829,12 @@ public class Table implements Index, Iterable<Row.Entry> {}@Override- public boolean delete(final byte[] key) throws IOException {+ public final boolean delete(final byte[] key) throws IOException {return remove(key) != null;}@Override- public synchronized Entry remove(final byte[] key) throws IOException {+ public final synchronized Entry remove(final byte[] key) throws IOException {assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();assert key.length == this.rowdef.primaryKeyLength;@@ -928,7 +929,7 @@ public class Table implements Index, Iterable<Row.Entry> {}@Override- public synchronized Entry removeOne() throws IOException {+ public final synchronized Entry removeOne() throws IOException {//assert this.file.size() == this.index.size() : "file.size() = " + this.file.size() + ", index.size() = " + this.index.size();assert this.table == null || this.table.size() == this.index.size() : "table.size() = " + this.table.size() + ", index.size() = " + this.index.size();final byte[] le = new byte[this.rowdef.objectsize];@@ -955,7 +956,7 @@ public class Table implements Index, Iterable<Row.Entry> {}@Override- public List<Row.Entry> top(int count) throws IOException {+ public final List<Row.Entry> top(int count) throws IOException {if (count > this.size()) count = this.size();final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>();if (this.file == null || this.index == null || this.size() == 0 || count == 0) return list;@@ -971,7 +972,7 @@ public class Table implements Index, Iterable<Row.Entry> {}@Override- public List<Row.Entry> random(int count) throws IOException {+ public final List<Row.Entry> random(int count) throws IOException {if (count > this.size()) count = this.size();final ArrayList<Row.Entry> list = new ArrayList<Row.Entry>();if (this.file == null || this.index == null || this.size() == 0 || count == 0) return list;@@ -996,23 +997,23 @@ public class Table implements Index, Iterable<Row.Entry> {}@Override- public Row row() {+ public final Row row() {return this.rowdef;}@Override- public int size() {+ public final int size() {if (this.index == null) return 0;return this.index.size();}@Override- public boolean isEmpty() {+ public final boolean isEmpty() {return this.index == null || this.index.isEmpty();}@Override- public Iterator<Entry> iterator() {+ public final Iterator<Entry> iterator() {try {return rows();} catch (final IOException e) {@@ -1026,7 +1027,7 @@ public class Table implements Index, Iterable<Row.Entry> {return new rowIteratorNoOrder();}- private class rowIteratorNoOrder implements CloneableIterator<Entry> {+ private final class rowIteratorNoOrder implements CloneableIterator<Entry> {Iterator<Map.Entry<byte[], Long>> i;long idx;byte[] key;@@ -1038,17 +1039,17 @@ public class Table implements Index, Iterable<Row.Entry> {}@Override- public CloneableIterator<Entry> clone(final Object modifier) {+ public final CloneableIterator<Entry> clone(final Object modifier) {return new rowIteratorNoOrder();}@Override- public boolean hasNext() {+ public final boolean hasNext() {return this.i != null && this.i.hasNext();}@Override- public Entry next() {+ public final Entry next() {final Map.Entry<byte[], Long> entry = this.i.next();if (entry == null) return null;this.key = entry.getKey();@@ -1062,7 +1063,7 @@ public class Table implements Index, Iterable<Row.Entry> {}@Override- public void remove() {+ public final void remove() {if (this.key != null) {try {removeInFile((int) this.idx);@@ -1074,7 +1075,7 @@ public class Table implements Index, Iterable<Row.Entry> {}@Override- public void close() {+ public final void close() {if (this.i instanceof CloneableIterator) {((CloneableIterator<Map.Entry<byte[], Long>>) this.i).close();}@@ -1087,7 +1088,7 @@ public class Table implements Index, Iterable<Row.Entry> {return new rowIterator(up, firstKey);}- private class rowIterator implements CloneableIterator<Entry> {+ private final class rowIterator implements CloneableIterator<Entry> {private final CloneableIterator<byte[]> i;private final boolean up;private final byte[] fk;@@ -1167,7 +1168,7 @@ public class Table implements Index, Iterable<Row.Entry> {for (int pos = 1; pos < source.length() - 1; pos++) {result[perm * source.length() + pos] = recres[perm].substring(0, pos) + c + recres[perm].substring(pos);}- result[perm * source.length() + source.length() - 1] = recres[perm] + c;+ result[perm * source.length() + source.length() - 1] = recres[perm] + c;}return result;}