Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Updated pdf cache clear steps consistently with current pdfbox version
luccioman54fbe166ba41
diff --git a/source/net/yacy/document/parser/pdfParser.java b/source/net/yacy/document/parser/pdfParser.javaindex ef3587646..bd3c3adfb 100644--- a/source/net/yacy/document/parser/pdfParser.java+++ b/source/net/yacy/document/parser/pdfParser.java@@ -32,7 +32,6 @@ import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io.IOException;import java.io.InputStream;-import java.lang.reflect.Method;import java.nio.charset.StandardCharsets;import java.util.ArrayList;import java.util.Collection;@@ -40,6 +39,7 @@ import java.util.Date;import java.util.HashSet;import java.util.List;+import org.apache.pdfbox.cos.COSName;import org.apache.pdfbox.io.MemoryUsageSetting;import org.apache.pdfbox.pdmodel.PDDocument;import org.apache.pdfbox.pdmodel.PDDocumentInformation;@@ -81,10 +81,6 @@ public class pdfParser extends AbstractParser implements Parser {this.SUPPORTED_MIME_TYPES.add("text/x-pdf");}- static {- clean_up_idiotic_PDFParser_font_cache_which_eats_up_tons_of_megabytes(); // must be called here to get that into the class loader; it will block other threads otherwise;- }-@Overridepublic Document[] parse(final DigestURL location,@@ -249,18 +245,9 @@ public class pdfParser extends AbstractParser implements Parser {try {pdfDoc.close();} catch (final Throwable e) {}}- // clear resources in pdfbox. they say that is resolved but it's not. see:- // https://issues.apache.org/jira/browse/PDFBOX-313- // https://issues.apache.org/jira/browse/PDFBOX-351- // https://issues.apache.org/jira/browse/PDFBOX-441- // the pdfbox still generates enormeous number of object allocations and don't delete these- // the following Object are statically stored and never flushed:- // COSFloat, COSArray, COSInteger, COSObjectKey, COSObject, COSDictionary,- // COSStream, COSString, COSName, COSDocument, COSInteger[], COSNull- // the great number of these objects can easily be seen in Java Visual VM- // we try to get this shit out of the memory here by forced clear calls, hope the best the rubbish gets out.+ // clear cached resources in pdfbox.pdfDoc = null;- clean_up_idiotic_PDFParser_font_cache_which_eats_up_tons_of_megabytes();+ clearPdfBoxCaches();return result;}@@ -295,55 +282,35 @@ public class pdfParser extends AbstractParser implements Parser {return linkCollections;}- public static void clean_up_idiotic_PDFParser_font_cache_which_eats_up_tons_of_megabytes() {- // thank you very much, PDFParser hackers, this font cache will occupy >80MB RAM for a single pdf and then stays forever- // AND I DO NOT EVEN NEED A FONT HERE TO PARSE THE TEXT!- // Don't be so ignorant, just google once "PDFParser OutOfMemoryError" to feel the pain.- ResourceCleaner cl = new ResourceCleaner();- cl.clearClassResources("org.apache.pdfbox.cos.COSName");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDFont");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDType1Font");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDTrueTypeFont");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDType0Font");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDType1AfmPfbFont");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDType3Font");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDType1CFont");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDCIDFont");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDCIDFontType0Font");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDCIDFontType2Font");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDMMType1Font");- cl.clearClassResources("org.apache.pdfbox.pdmodel.font.PDSimpleFont");+ /**+ * Clean up cache resources allocated by PDFBox that would otherwise not be released.+ */+ public static void clearPdfBoxCaches() {+ /*+ * Prior to pdfbox 2.0.0 font cache occupied > 80MB RAM for a single pdf and+ * then stayed forever (detected in YaCy with pdfbox version 1.2.1). The+ * situation is now from far better, but one (unnecessary?) cache structure in+ * the COSName class still needs to be explicitely cleared.+ */++ // History of related issues :+ // http://markmail.org/thread/quk5odee4hbsauhu+ // https://issues.apache.org/jira/browse/PDFBOX-313+ // https://issues.apache.org/jira/browse/PDFBOX-351+ // https://issues.apache.org/jira/browse/PDFBOX-441+ // https://issues.apache.org/jira/browse/PDFBOX-2200+ // https://issues.apache.org/jira/browse/PDFBOX-2149++ COSName.clearResources();++ /*+ * Prior to PDFBox 2.0.0, clearResources() function had to be called on the+ * org.apache.pdfbox.pdmodel.font.PDFont class and its children. After version+ * 2.0.0, there is no more such a function in PDFont class as font cache is+ * handled differently and hopefully more properly.+ */}- @SuppressWarnings({ "unchecked", "rawtypes" })- private static class ResourceCleaner {- Method findLoadedClass;- private ClassLoader sys;- public ResourceCleaner() {- try {- this.findLoadedClass = ClassLoader.class.getDeclaredMethod("findLoadedClass", new Class[] { String.class });- this.findLoadedClass.setAccessible(true);- this.sys = ClassLoader.getSystemClassLoader();- } catch (Throwable e) {- e.printStackTrace();- this.findLoadedClass = null;- this.sys = null;- }- }- public void clearClassResources(String name) {- if (this.findLoadedClass == null) return;- try {- Object pdfparserpainclass = this.findLoadedClass.invoke(this.sys, name);- if (pdfparserpainclass != null) {- Method clearResources = ((Class) pdfparserpainclass).getDeclaredMethod("clearResources", new Class[] {});- if (clearResources != null) clearResources.invoke(null);- }- } catch (Throwable e) {- //e.printStackTrace();- }- }- }-/*** test* @param argsdiff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.javaindex 93b4e4b7c..19b6223cf 100644--- a/source/net/yacy/search/Switchboard.java+++ b/source/net/yacy/search/Switchboard.java@@ -2454,7 +2454,7 @@ public final class Switchboard extends serverSwitch {public static void clearCaches() {// flush caches in used libraries- pdfParser.clean_up_idiotic_PDFParser_font_cache_which_eats_up_tons_of_megabytes(); // eats up megabytes, see http://markmail.org/thread/quk5odee4hbsauhu+ pdfParser.clearPdfBoxCaches();// clear cachesif (WordCache.sizeCommonWords() > 1000) WordCache.clearCommonWords();