Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
removed/suppressed more warnings
Michael Peter Christen1893661ee4aa
diff --git a/source/net/yacy/cora/util/Html2Image.java b/source/net/yacy/cora/util/Html2Image.javaindex baf8d76be..8e437a2fb 100644--- a/source/net/yacy/cora/util/Html2Image.java+++ b/source/net/yacy/cora/util/Html2Image.java@@ -43,6 +43,10 @@ import javax.swing.text.html.HTMLDocument;import javax.swing.text.html.HTMLEditorKit;import javax.swing.text.html.ImageView;+import org.apache.pdfbox.pdmodel.PDDocument;+import org.apache.pdfbox.rendering.ImageType;+import org.apache.pdfbox.rendering.PDFRenderer;+import net.yacy.cora.document.id.MultiProtocolURL;import net.yacy.cora.protocol.ClientIdentification;import net.yacy.cora.protocol.Domains;@@ -51,57 +55,53 @@ import net.yacy.document.ImageParser;import net.yacy.kelondro.util.FileUtils;import net.yacy.kelondro.util.OS;-import org.apache.pdfbox.pdmodel.PDDocument;-import org.apache.pdfbox.rendering.ImageType;-import org.apache.pdfbox.rendering.PDFRenderer;-/*** Convert html to an copy on disk-image in a other file format* currently (pdf and/or jpg)*/public class Html2Image {-+// Mac- /**- * Path to wkhtmltopdf executable on Mac OS when installed using- * wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html.- * This can also be a path on Debian or another Gnu/Linux distribution.- */- private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf");-+ /**+ * Path to wkhtmltopdf executable on Mac OS when installed using+ * wkhtmltox-n.n.n.macos-cocoa.pkg from https://wkhtmltopdf.org/downloads.html.+ * This can also be a path on Debian or another Gnu/Linux distribution.+ */+ private final static File wkhtmltopdfMac = new File("/usr/local/bin/wkhtmltopdf");+// to install imagemagick, download from http://cactuslab.com/imagemagick/assets/ImageMagick-6.8.9-9.pkg.zip// the convert command from imagemagick needs ghostscript, if not present on older macs, download a version of gs from http://pages.uoregon.edu/koch/-+private final static File convertMac1 = new File("/opt/local/bin/convert");private final static File convertMac2 = new File("/opt/ImageMagick/bin/convert");-+/* Debian packages to install: apt-get install wkhtmltopdf imagemagick xvfb ghostscriptThe imagemagick policy at /etc should also be checked :if it contains a line such as <policy domain="coder" rights="none" pattern="PDF" /> it must be edited with rights="read" at minimum- */+ */private final static File wkhtmltopdfDebian = new File("/usr/bin/wkhtmltopdf"); // there is no wkhtmltoimage, use convert to create imagesprivate final static File convertDebian = new File("/usr/bin/convert");-- /**- * Path to wkhtmltopdf executable on Windows, when installed with default- * settings using wkhtmltox-n.n.n.msvc2015-win64.exe from- * https://wkhtmltopdf.org/downloads.html- */- private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");-- /**- * Path to wkhtmltopdf executable on Windows, when installed with default- * settings using wkhtmltox-n.n.n.msvc2015-win32.exe from- * https://wkhtmltopdf.org/downloads.html- */- private static final File WKHTMLTOPDF_WINDOWS_X86 = new File(- "C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");-- /** Command to use when wkhtmltopdf is included in the system Path */- private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf";-- /** Command to use when imagemagick convert is included in the system Path */- private static final String CONVERT_COMMAND = "convert";++ /**+ * Path to wkhtmltopdf executable on Windows, when installed with default+ * settings using wkhtmltox-n.n.n.msvc2015-win64.exe from+ * https://wkhtmltopdf.org/downloads.html+ */+ private static final File WKHTMLTOPDF_WINDOWS = new File("C:\\Program Files\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");++ /**+ * Path to wkhtmltopdf executable on Windows, when installed with default+ * settings using wkhtmltox-n.n.n.msvc2015-win32.exe from+ * https://wkhtmltopdf.org/downloads.html+ */+ private static final File WKHTMLTOPDF_WINDOWS_X86 = new File(+ "C:\\Program Files (x86)\\wkhtmltopdf\\bin\\wkhtmltopdf.exe");++ /** Command to use when wkhtmltopdf is included in the system Path */+ private static final String WKHTMLTOPDF_COMMAND = "wkhtmltopdf";++ /** Command to use when imagemagick convert is included in the system Path */+ private static final String CONVERT_COMMAND = "convert";private static boolean usexvfb = false;@@ -109,109 +109,111 @@ public class Html2Image {* @return when the wkhtmltopdf command is detected as available in the system*/public static boolean wkhtmltopdfAvailable() {- /* Check wkhtmltopdf common installation paths and system Path */- return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath();+ /* Check wkhtmltopdf common installation paths and system Path */+ return wkhtmltopdfExecutable() != null || wkhtmltopdfAvailableInPath();}-+/*** @return a wkhtmltopdf executable file when one can be found, null otherwise*/private static File wkhtmltopdfExecutable() {- File executable = null;- if(OS.isWindows) {- if(WKHTMLTOPDF_WINDOWS.exists()) {- executable = WKHTMLTOPDF_WINDOWS;- } else if(WKHTMLTOPDF_WINDOWS_X86.exists()) {- executable = WKHTMLTOPDF_WINDOWS_X86;- }- } else {- if(wkhtmltopdfMac.exists()) {- executable = wkhtmltopdfMac;- } else if(wkhtmltopdfDebian.exists()) {- executable = wkhtmltopdfDebian;- }- }- return executable;+ File executable = null;+ if(OS.isWindows) {+ if(WKHTMLTOPDF_WINDOWS.exists()) {+ executable = WKHTMLTOPDF_WINDOWS;+ } else if(WKHTMLTOPDF_WINDOWS_X86.exists()) {+ executable = WKHTMLTOPDF_WINDOWS_X86;+ }+ } else {+ if(wkhtmltopdfMac.exists()) {+ executable = wkhtmltopdfMac;+ } else if(wkhtmltopdfDebian.exists()) {+ executable = wkhtmltopdfDebian;+ }+ }+ return executable;}/*** @return true when wkhtmltopdf is available in system path*/- private static boolean wkhtmltopdfAvailableInPath() {- boolean available = false;- try {- final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V");- available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;- } catch (final IOException e) {- ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path.");- } catch (final InterruptedException e) {- Thread.currentThread().interrupt(); // preserve thread interrupted state- }- return available;- }-+ private static boolean wkhtmltopdfAvailableInPath() {+ boolean available = false;+ try {+ @SuppressWarnings("deprecation")+ final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V");+ available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;+ } catch (final IOException e) {+ ConcurrentLog.fine("Html2Image", "wkhtmltopdf is not included in system path.");+ } catch (final InterruptedException e) {+ Thread.currentThread().interrupt(); // preserve thread interrupted state+ }+ return available;+ }+/*** @return a imagemagick convert executable file when one can be found, null otherwise*/private static File convertExecutable() {- File executable = null;- if(!OS.isWindows) {- if(convertMac1.exists()) {- executable = convertMac1;- } else if(convertMac2.exists()) {- executable = convertMac2;- } else if(convertDebian.exists()) {- executable = convertDebian;- }- }- return executable;+ File executable = null;+ if(!OS.isWindows) {+ if(convertMac1.exists()) {+ executable = convertMac1;+ } else if(convertMac2.exists()) {+ executable = convertMac2;+ } else if(convertDebian.exists()) {+ executable = convertDebian;+ }+ }+ return executable;}-+/*** @return when the imagemagick convert command is detected as available in the system*/public static boolean convertAvailable() {- /* Check convert common installation paths and system Path */+ /* Check convert common installation paths and system Path */return convertExecutable() != null || convertAvailableInPath();}/*** @return when imagemagick convert is available in system path*/- private static boolean convertAvailableInPath() {- boolean available = false;- if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS- try {- final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version");- available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;- } catch (final IOException e) {- ConcurrentLog.fine("Html2Image", "convert is not included in system path.");- } catch (final InterruptedException e) {- Thread.currentThread().interrupt(); // preserve thread interrupted state- }- }- return available;- }-- /**- * Run the wkhtmltopdf external tool to fetch and render to PDF a web resource.- * wKhtmltopdf may be called multiple times with various parameters flavors in- * case of failure.- *- * @param url the URL of a web resource to fetch, render and convert to- * a pdf file. Must not be null.- * @param proxy the eventual proxy address to use. Can be null. Must be of- * the form http://host:port; use YaCy here as proxy which is- * mostly http://localhost:8090- * @param destination the destination PDF file that should be written. Must not- * be null.- * @param maxSeconds the maximum time in seconds to wait for each wkhtmltopdf- * call termination. Beyond this limit the process is killed.- * @return true when the destination file was successfully written- */+ private static boolean convertAvailableInPath() {+ boolean available = false;+ if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS+ try {+ @SuppressWarnings("deprecation")+ final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version");+ available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0;+ } catch (final IOException e) {+ ConcurrentLog.fine("Html2Image", "convert is not included in system path.");+ } catch (final InterruptedException e) {+ Thread.currentThread().interrupt(); // preserve thread interrupted state+ }+ }+ return available;+ }++ /**+ * Run the wkhtmltopdf external tool to fetch and render to PDF a web resource.+ * wKhtmltopdf may be called multiple times with various parameters flavors in+ * case of failure.+ *+ * @param url the URL of a web resource to fetch, render and convert to+ * a pdf file. Must not be null.+ * @param proxy the eventual proxy address to use. Can be null. Must be of+ * the form http://host:port; use YaCy here as proxy which is+ * mostly http://localhost:8090+ * @param destination the destination PDF file that should be written. Must not+ * be null.+ * @param maxSeconds the maximum time in seconds to wait for each wkhtmltopdf+ * call termination. Beyond this limit the process is killed.+ * @return true when the destination file was successfully written+ */public static boolean writeWkhtmltopdf(String url, String proxy, String userAgent, final String acceptLanguage, final File destination, final long maxSeconds) {boolean success = false;- for (boolean ignoreErrors: new boolean[]{false, true}) {+ for (final boolean ignoreErrors: new boolean[]{false, true}) {success = writeWkhtmltopdfInternal(url, proxy, destination, userAgent, acceptLanguage, ignoreErrors, maxSeconds);if (success) break;if (!success && proxy != null) {@@ -227,49 +229,49 @@ public class Html2Image {}return success;}-- /**- * Run wkhtmltopdf in a separate process to fetch and render to PDF a web- * resource.- *- * @param url the URL of a web resource to fetch, render and convert to- * a pdf file. Must not be null.- * @param proxy the eventual proxy address to use. Can be null.- * @param destination the destination PDF file that should be written. Must not- * be null.- * @param userAgent TODO: implement- * @param acceptLanguage TODO: implement- * @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors- * @param maxSeconds the maximum time in seconds to wait for the wkhtmltopdf- * dedicated process termination. Beyond this limit the- * process is killed.- * @return true when the destination file was successfully written- */- private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination,- final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) {- final String wkhtmltopdfCmd;- final File wkhtmltopdf = wkhtmltopdfExecutable();- if(wkhtmltopdf != null) {- wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath();- } else if(wkhtmltopdfAvailableInPath()) {- wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND;- } else {- ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!");- return false;- }++ /**+ * Run wkhtmltopdf in a separate process to fetch and render to PDF a web+ * resource.+ *+ * @param url the URL of a web resource to fetch, render and convert to+ * a pdf file. Must not be null.+ * @param proxy the eventual proxy address to use. Can be null.+ * @param destination the destination PDF file that should be written. Must not+ * be null.+ * @param userAgent TODO: implement+ * @param acceptLanguage TODO: implement+ * @param ignoreErrors when true wkhtmltopdf is instructed to ignore load errors+ * @param maxSeconds the maximum time in seconds to wait for the wkhtmltopdf+ * dedicated process termination. Beyond this limit the+ * process is killed.+ * @return true when the destination file was successfully written+ */+ private static boolean writeWkhtmltopdfInternal(final String url, final String proxy, final File destination,+ final String userAgent, final String acceptLanguage, final boolean ignoreErrors, final long maxSeconds) {+ final String wkhtmltopdfCmd;+ final File wkhtmltopdf = wkhtmltopdfExecutable();+ if(wkhtmltopdf != null) {+ wkhtmltopdfCmd = wkhtmltopdf.getAbsolutePath();+ } else if(wkhtmltopdfAvailableInPath()) {+ wkhtmltopdfCmd = WKHTMLTOPDF_COMMAND;+ } else {+ ConcurrentLog.warn("Html2Pdf", "Unable to locate wkhtmltopdf executable on this system!");+ return false;+ }String commandline =- wkhtmltopdfCmd + " -q --title '" + url + "' " +- //acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " +- //(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") +- (proxy == null ? "" : "--proxy " + proxy + " ") +- (ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it...- //"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " +- "--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " +- url + " " + destination.getAbsolutePath();+ wkhtmltopdfCmd + " -q --title '" + url + "' " ++ //acceptLanguage == null ? "" : "--custom-header 'Accept-Language' '" + acceptLanguage + "' " ++ //(userAgent == null ? "" : "--custom-header \"User-Agent\" \"" + userAgent + "\" --custom-header-propagation ") ++ (proxy == null ? "" : "--proxy " + proxy + " ") ++ (ignoreErrors ? (OS.isMacArchitecture ? "--load-error-handling ignore " : "--ignore-load-errors ") : "") + // some versions do not have that flag and fail if attempting to use it...+ //"--footer-font-name 'Courier' --footer-font-size 9 --footer-left [webpage] --footer-right [date]/[time]([page]/[topage]) " ++ "--footer-left [webpage] --footer-right '[date]/[time]([page]/[topage])' --footer-font-size 7 " ++ url + " " + destination.getAbsolutePath();try {- ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline);+ ConcurrentLog.info("Html2Pdf", "creating pdf from url " + url + " with command: " + commandline);if (!usexvfb && execWkhtmlToPdf(proxy, destination, commandline, maxSeconds)) {- return true;+ return true;}// if this fails, we should try to wrap the X server with a virtual screen using xvfb, this works on headless serverscommandline = "xvfb-run -a " + commandline;@@ -280,46 +282,46 @@ public class Html2Image {}}- /**- * Run a wkhtmltopdf commandline in a separate process.- *- * @param proxy the eventual proxy address to use. Can be null.- * @param destination the destination PDF file that should be written. Must not- * be null.- * @param commandline the wkhtmltopdf command line to execute. Must not be null.- * @param maxSeconds the maximum time in seconds to wait for the process- * termination. Beyond this limit the process is killed.- * @return true when the destination file was successfully written- * @throws IOException when an unexpected error occurred- */- private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds)- throws IOException {- final Process p = Runtime.getRuntime().exec(commandline);-- try {- p.waitFor(maxSeconds, TimeUnit.SECONDS);- } catch (final InterruptedException e) {- p.destroyForcibly();- ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline);- Thread.currentThread().interrupt(); // Keep the thread interrupted state- return false;- }- if(p.isAlive()) {- ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline);- p.destroyForcibly();- return false;- }- if (p.exitValue() == 0 && destination.exists()) {- return true;- }- final List<String> messages = OS.readStreams(p);- ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline);- for (final String message : messages) {- ConcurrentLog.warn("Html2Image", ">> " + message);- }- return false;- }-+ /**+ * Run a wkhtmltopdf commandline in a separate process.+ *+ * @param proxy the eventual proxy address to use. Can be null.+ * @param destination the destination PDF file that should be written. Must not+ * be null.+ * @param commandline the wkhtmltopdf command line to execute. Must not be null.+ * @param maxSeconds the maximum time in seconds to wait for the process+ * termination. Beyond this limit the process is killed.+ * @return true when the destination file was successfully written+ * @throws IOException when an unexpected error occurred+ */+ private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds) throws IOException {+ @SuppressWarnings("deprecation")+ final Process p = Runtime.getRuntime().exec(commandline);++ try {+ p.waitFor(maxSeconds, TimeUnit.SECONDS);+ } catch (final InterruptedException e) {+ p.destroyForcibly();+ ConcurrentLog.warn("Html2Pdf", "Interrupted creation of pdf. Killing the process started with command : " + commandline);+ Thread.currentThread().interrupt(); // Keep the thread interrupted state+ return false;+ }+ if(p.isAlive()) {+ ConcurrentLog.warn("Html2Pdf", "Creation of pdf did not terminate within " + maxSeconds + " seconds. Killing the process started with command : " + commandline);+ p.destroyForcibly();+ return false;+ }+ if (p.exitValue() == 0 && destination.exists()) {+ return true;+ }+ final List<String> messages = OS.readStreams(p);+ ConcurrentLog.warn("Html2Image", "failed to create pdf " + (proxy == null ? "" : "using proxy " + proxy) + " with command : " + commandline);+ for (final String message : messages) {+ ConcurrentLog.warn("Html2Image", ">> " + message);+ }+ return false;+ }+/*** Convert a pdf (first page) to an image. Proper values are i.e. width = 1024, height = 1024, density = 300, quality = 75* using internal pdf library or external command line tool on linux or mac@@ -332,34 +334,34 @@ public class Html2Image {* @return true when the ouput image file was successfully written.*/public static boolean pdf2image(final File pdf, final File image, final int width, final int height, final int density, final int quality) {- /* Deduce the ouput image format from the file extension */- String imageFormat = MultiProtocolURL.getFileExtension(image.getName());- if(imageFormat.isEmpty()) {- /* Use JPEG as a default fallback */- imageFormat = "jpg";- }- String convertCmd = null;- final File convert = convertExecutable();- if(convert != null) {- convertCmd = convert.getAbsolutePath();- } else if(convertAvailableInPath()) {- convertCmd = CONVERT_COMMAND;- } else {- ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!");- }+ /* Deduce the ouput image format from the file extension */+ String imageFormat = MultiProtocolURL.getFileExtension(image.getName());+ if(imageFormat.isEmpty()) {+ /* Use JPEG as a default fallback */+ imageFormat = "jpg";+ }+ String convertCmd = null;+ final File convert = convertExecutable();+ if(convert != null) {+ convertCmd = convert.getAbsolutePath();+ } else if(convertAvailableInPath()) {+ convertCmd = CONVERT_COMMAND;+ } else {+ ConcurrentLog.info("Html2Image", "Unable to locate convert executable on this system!");+ }// convert pdf to jpg using internal pdfbox capabilityif (convertCmd == null) {try (final PDDocument pdoc = PDDocument.load(pdf);) {-- BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB);++ final BufferedImage bi = new PDFRenderer(pdoc).renderImageWithDPI(0, density, ImageType.RGB);return ImageIO.write(bi, imageFormat, image);} catch (final IOException ex) {- ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox"- + (ex.getMessage() != null ? " : " + ex.getMessage() : ""));- return false;+ ConcurrentLog.warn("Html2Image", "Failed to create image with pdfbox"+ + (ex.getMessage() != null ? " : " + ex.getMessage() : ""));+ return false;}}@@ -367,18 +369,18 @@ public class Html2Image {try {// i.e. convert -density 300 -trim yacy.pdf[0] -trim -resize 1024x -crop x1024+0+0 -quality 75% yacy-convert-300.jpg// note: both -trim are necessary, otherwise it is trimmed only on one side. The [0] selects the first page of the pdf- String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath();+ final String command = convertCmd + " -alpha remove -density " + density + " -trim " + pdf.getAbsolutePath() + "[0] -trim -resize " + width + "x -crop x" + height + "+0+0 -quality " + quality + "% " + image.getAbsolutePath();List<String> message = OS.execSynchronous(command);if (image.exists()) return true;ConcurrentLog.warn("Html2Image", "failed to create image with command: " + command);- for (String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);-+ for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);+// another try for mac: use Image Events using AppleScript in osacript commands...// the following command overwrites a pdf with an png, so we must make a copy firstif (!OS.isMacArchitecture) return false;- File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf");+ final File pngFile = new File(pdf.getAbsolutePath() + ".tmp.pdf");org.apache.commons.io.FileUtils.copyFile(pdf, pngFile);- String[] commandx = {"osascript",+ final String[] commandx = {"osascript","-e", "set ImgFile to \"" + pngFile.getAbsolutePath() + "\"","-e", "tell application \"Image Events\"","-e", "set Img to open file ImgFile",@@ -386,15 +388,15 @@ public class Html2Image {"-e", "end tell"};//ConcurrentLog.warn("Html2Image", "failed to create image with command: " + commandx);message = OS.execSynchronous(commandx);- for (String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);+ for (final String m: message) ConcurrentLog.warn("Html2Image", ">> " + m);// now we must read and convert this file to the target format with the target size 1024x1024try {- File newPngFile = new File(pngFile.getAbsolutePath() + ".png");+ final File newPngFile = new File(pngFile.getAbsolutePath() + ".png");pngFile.renameTo(newPngFile);final Image img = ImageParser.parse(pngFile.getAbsolutePath(), FileUtils.read(newPngFile));if(img == null) {- /* Should not happen. If so, ImageParser.parse() should already have logged about the error */- return false;+ /* Should not happen. If so, ImageParser.parse() should already have logged about the error */+ return false;}final Image scaled = img.getScaledInstance(width, height, Image.SCALE_AREA_AVERAGING);final MediaTracker mediaTracker = new MediaTracker(new Container());@@ -406,24 +408,24 @@ public class Html2Image {ImageIO.write(bi, imageFormat, image);newPngFile.delete();return image.exists();- } catch (IOException e) {+ } catch (final IOException e) {ConcurrentLog.logException(e);return false;}- } catch (IOException e) {+ } catch (final IOException e) {e.printStackTrace();return false;}}-+/*** render a html page with a JEditorPane, which can do html up to html v 3.2. No CSS supported!* @param url* @param size- * @throws IOException+ * @throws IOException*/public static void writeSwingImage(String url, Dimension size, File destination) throws IOException {-+// set up a pane for renderingfinal JEditorPane htmlPane = new JEditorPane();htmlPane.setSize(size);@@ -434,7 +436,7 @@ public class Html2Image {@Overridepublic Document createDefaultDocument() {- HTMLDocument doc = (HTMLDocument) super.createDefaultDocument();+ final HTMLDocument doc = (HTMLDocument) super.createDefaultDocument();doc.setAsynchronousLoadPriority(-1);return doc;}@@ -444,7 +446,7 @@ public class Html2Image {return new HTMLFactory() {@Overridepublic View create(Element elem) {- View view = super.create(elem);+ final View view = super.create(elem);if (view instanceof ImageView) {((ImageView) view).setLoadsSynchronously(true);}@@ -460,23 +462,23 @@ public class Html2Image {public void propertyChange(PropertyChangeEvent evt) {}});-+// load the pagetry {htmlPane.setPage(url);- } catch (IOException e) {+ } catch (final IOException e) {e.printStackTrace();}-+// render the page- Dimension prefSize = htmlPane.getPreferredSize();- BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB);- Graphics graphics = img.getGraphics();+ final Dimension prefSize = htmlPane.getPreferredSize();+ final BufferedImage img = new BufferedImage(prefSize.width, htmlPane.getPreferredSize().height, BufferedImage.TYPE_INT_ARGB);+ final Graphics graphics = img.getGraphics();htmlPane.setSize(prefSize);htmlPane.paint(graphics);ImageIO.write(img, destination.getName().endsWith("jpg") ? "jpg" : "png", destination);}-+/*** Test PDF or image snapshot generation for a given URL.* @param args main arguments list:@@ -491,88 +493,88 @@ public class Html2Image {* </li>* </ol>*/- public static void main(String[] args) {- final String usageMessage = "Usage : java " + Html2Image.class.getName()- + " <url> <target-file[.pdf|.jpg|.png]> [wkhtmltopdf|swing]";- int exitStatus = 0;- try {- if (args.length < 2) {- System.out.println("Missing required parameter(s).");- System.out.println(usageMessage);- exitStatus = 1;- return;- }- final String targetPath = args[1];- if (args.length < 3 || "wkhtmltopdf".equals(args[2])) {- if(Html2Image.wkhtmltopdfAvailable()) {- final File targetPdfFile;- if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {- targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf");- } else if(targetPath.endsWith(".pdf")) {- targetPdfFile = new File(targetPath);- } else {- System.out.println("Unsupported output format");- System.out.println(usageMessage);- exitStatus = 1;- return;- }- if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent,- "en-us,en;q=0.5", targetPdfFile, 30)) {- if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {- if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) {- ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile);- } else {- exitStatus = 1;- return;- }- }- } else {- exitStatus = 1;- return;- }- } else {- System.out.println("Unable to locate wkhtmltopdf executable on this system!");- exitStatus = 1;- return;- }- } else if ("swing".equals(args[2])) {- if(targetPath.endsWith(".pdf")) {- System.out.println("Pdf output format is not supported with swing method.");- exitStatus = 1;- return;- }- if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) {- System.out.println("Unsupported output format");- System.out.println(usageMessage);- exitStatus = 1;- return;- }-- try {- Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath));- } catch (final IOException e) {- e.printStackTrace();- exitStatus = 1;- return;- }- } else {- System.out.println("Unknown method : please specify either wkhtmltopdf or swing.");- exitStatus = 1;- return;- }- } finally {- /* Shutdown running threads */- Domains.close();- try {- HTTPClient.closeConnectionManager();- } catch (final InterruptedException e) {- Thread.currentThread().interrupt(); // restore interrupted state- }- ConcurrentLog.shutdown();- if(exitStatus != 0) {- System.exit(exitStatus);- }- }- }-+ public static void main(String[] args) {+ final String usageMessage = "Usage : java " + Html2Image.class.getName()+ + " <url> <target-file[.pdf|.jpg|.png]> [wkhtmltopdf|swing]";+ int exitStatus = 0;+ try {+ if (args.length < 2) {+ System.out.println("Missing required parameter(s).");+ System.out.println(usageMessage);+ exitStatus = 1;+ return;+ }+ final String targetPath = args[1];+ if (args.length < 3 || "wkhtmltopdf".equals(args[2])) {+ if(Html2Image.wkhtmltopdfAvailable()) {+ final File targetPdfFile;+ if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {+ targetPdfFile = new File(targetPath.substring(0, targetPath.length() - 4) + ".pdf");+ } else if(targetPath.endsWith(".pdf")) {+ targetPdfFile = new File(targetPath);+ } else {+ System.out.println("Unsupported output format");+ System.out.println(usageMessage);+ exitStatus = 1;+ return;+ }+ if(Html2Image.writeWkhtmltopdf(args[0], null, ClientIdentification.yacyInternetCrawlerAgent.userAgent,+ "en-us,en;q=0.5", targetPdfFile, 30)) {+ if(targetPath.endsWith(".jpg") || targetPath.endsWith(".png")) {+ if(Html2Image.pdf2image(targetPdfFile, new File(targetPath), 1024, 1024, 300, 75)) {+ ConcurrentLog.info("Html2Image", "wrote " + targetPath + " converted from " + targetPdfFile);+ } else {+ exitStatus = 1;+ return;+ }+ }+ } else {+ exitStatus = 1;+ return;+ }+ } else {+ System.out.println("Unable to locate wkhtmltopdf executable on this system!");+ exitStatus = 1;+ return;+ }+ } else if ("swing".equals(args[2])) {+ if(targetPath.endsWith(".pdf")) {+ System.out.println("Pdf output format is not supported with swing method.");+ exitStatus = 1;+ return;+ }+ if(!targetPath.endsWith(".jpg") && !targetPath.endsWith(".png")) {+ System.out.println("Unsupported output format");+ System.out.println(usageMessage);+ exitStatus = 1;+ return;+ }++ try {+ Html2Image.writeSwingImage(args[0], new Dimension(1200, 2000), new File(targetPath));+ } catch (final IOException e) {+ e.printStackTrace();+ exitStatus = 1;+ return;+ }+ } else {+ System.out.println("Unknown method : please specify either wkhtmltopdf or swing.");+ exitStatus = 1;+ return;+ }+ } finally {+ /* Shutdown running threads */+ Domains.close();+ try {+ HTTPClient.closeConnectionManager();+ } catch (final InterruptedException e) {+ Thread.currentThread().interrupt(); // restore interrupted state+ }+ ConcurrentLog.shutdown();+ if(exitStatus != 0) {+ System.exit(exitStatus);+ }+ }+ }+}diff --git a/source/net/yacy/cora/util/Memory.java b/source/net/yacy/cora/util/Memory.javaindex 3a1334d91..eb47f0908 100644--- a/source/net/yacy/cora/util/Memory.java+++ b/source/net/yacy/cora/util/Memory.java@@ -115,8 +115,9 @@ public class Memory {* @return the "recent cpu usage" for the whole operating environment;* a negative value if not available.*/+ @SuppressWarnings("deprecation")public static double getSystemCpuLoad() {- com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();+ final com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();return operatingSystemMXBean.getSystemCpuLoad();}@@ -136,13 +137,13 @@ public class Memory {* a negative value if not available.*/public static double getProcessCpuLoad() {- com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();+ final com.sun.management.OperatingSystemMXBean operatingSystemMXBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();return operatingSystemMXBean.getProcessCpuLoad();}public static Map<String, Object> status() {- Runtime runtime = Runtime.getRuntime();- Map<String, Object> status = new LinkedHashMap<>();+ final Runtime runtime = Runtime.getRuntime();+ final Map<String, Object> status = new LinkedHashMap<>();status.put("service", "Peer");status.put("assigned_memory", runtime.maxMemory());status.put("used_memory", runtime.totalMemory() - runtime.freeMemory());@@ -153,7 +154,7 @@ public class Memory {status.put("load_system_load_average", Memory.getSystemLoadAverage());status.put("load_system_cpu_load", Memory.getSystemCpuLoad());status.put("load_process_cpu_load", Memory.getProcessCpuLoad());- YaCyHttpServer server = Switchboard.getSwitchboard().getHttpServer();+ final YaCyHttpServer server = Switchboard.getSwitchboard().getHttpServer();status.put("server_threads", server == null ? 0 : server.getServerThreads());return status;}@@ -163,7 +164,7 @@ public class Memory {* @return the number of deadlocked threads*/public static long deadlocks() {- long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();+ final long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();if (deadlockIDs == null) return 0;return deadlockIDs.length;}@@ -172,10 +173,10 @@ public class Memory {* write deadlocked threads as to the log as warning*/public static void logDeadlocks() {- long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();+ final long[] deadlockIDs = ManagementFactory.getThreadMXBean().findDeadlockedThreads();if (deadlockIDs == null) return;- ThreadInfo[] infos = ManagementFactory.getThreadMXBean().getThreadInfo(deadlockIDs, true, true);- for (ThreadInfo ti : infos) {+ final ThreadInfo[] infos = ManagementFactory.getThreadMXBean().getThreadInfo(deadlockIDs, true, true);+ for (final ThreadInfo ti : infos) {ConcurrentLog.warn("DEADLOCKREPORT", ti.toString());}}diff --git a/source/net/yacy/data/list/XMLBlacklistImporter.java b/source/net/yacy/data/list/XMLBlacklistImporter.javaindex 6f72f091d..20155df3c 100644--- a/source/net/yacy/data/list/XMLBlacklistImporter.java+++ b/source/net/yacy/data/list/XMLBlacklistImporter.java@@ -33,7 +33,6 @@ import org.xml.sax.InputSource;import org.xml.sax.SAXException;import org.xml.sax.XMLReader;import org.xml.sax.helpers.DefaultHandler;-import org.xml.sax.helpers.XMLReaderFactory;/*** This class provides methods to import blacklists from an XML file (see@@ -54,13 +53,15 @@ public class XMLBlacklistImporter extends DefaultHandler {* @throws java.io.IOException if input can't be read* @throws org.xml.sax.SAXException if XML can't be parsed*/+ @SuppressWarnings("deprecation")public synchronized ListAccumulator parse(InputSource input) throws IOException, SAXException {- XMLReader reader = XMLReaderFactory.createXMLReader();+ @SuppressWarnings("deprecation")+ final XMLReader reader = org.xml.sax.helpers.XMLReaderFactory.createXMLReader();reader.setContentHandler(this);reader.parse(input);- return ba;+ return this.ba;}/**@@ -104,7 +105,7 @@ public class XMLBlacklistImporter extends DefaultHandler {*/@Overridepublic void startDocument() {- ba = new ListAccumulator();+ this.ba = new ListAccumulator();}/**@@ -127,24 +128,24 @@ public class XMLBlacklistImporter extends DefaultHandler {public void startElement(final String uri, final String localName, final String qName, final Attributes attributes) {if (qName.equalsIgnoreCase("list")) {- currentListName = attributes.getValue("name");- ba.addList(currentListName);-+ this.currentListName = attributes.getValue("name");+ this.ba.addList(this.currentListName);+int attributesLength = 0;if ((attributesLength = attributes.getLength()) > 1) {for (int i = 0; i < attributesLength; i++) {if (!attributes.getQName(i).equals("name")) {- ba.addPropertyToCurrent(attributes.getQName(i), attributes.getValue(i));+ this.ba.addPropertyToCurrent(attributes.getQName(i), attributes.getValue(i));}}}}-+if (qName.equalsIgnoreCase("item")) {- lastText = new StringBuilder();+ this.lastText = new StringBuilder();}-+}/**@@ -162,7 +163,7 @@ public class XMLBlacklistImporter extends DefaultHandler {@Overridepublic void endElement(final String uri, final String localName, final String qName) throws SAXException {if (qName.equalsIgnoreCase("item")) {- ba.addEntryToCurrent(lastText.toString());+ this.ba.addEntryToCurrent(this.lastText.toString());}}@@ -175,8 +176,8 @@ public class XMLBlacklistImporter extends DefaultHandler {*/@Overridepublic void characters(char[] ch, int start, int length) throws SAXException {- if (lastText == null) lastText = new StringBuilder();- lastText.append(ch, start, length);+ if (this.lastText == null) this.lastText = new StringBuilder();+ this.lastText.append(ch, start, length);}}diff --git a/source/net/yacy/gui/framework/Browser.java b/source/net/yacy/gui/framework/Browser.javaindex 0b967c6ac..741a30047 100644--- a/source/net/yacy/gui/framework/Browser.java+++ b/source/net/yacy/gui/framework/Browser.java@@ -37,7 +37,7 @@ import net.yacy.cora.util.ConcurrentLog;public class Browser {- // constants for system identification+ // constants for system identificationpublic static final int systemMacOSC = 0; // 'classic' Mac OS 7.6.1/8.*/9.*public static final int systemMacOSX = 1; // all Mac OS Xpublic static final int systemUnix = 2; // all Unix/Linux type systems@@ -100,7 +100,7 @@ public class Browser {openBrowserClassic(url);}}-+public static void openBrowserClassic(final String url) {try {if (systemOS == systemMacOSX) {@@ -113,12 +113,12 @@ public class Browser {throw new RuntimeException("System unknown");}} catch (final Throwable e) {- ConcurrentLog.warn("BROWSER", "Could not open browser : " + e.getMessage() != null ? e.getMessage() : e.toString());+ ConcurrentLog.warn("BROWSER", "Could not open browser : " + e.getMessage() != null ? e.getMessage() : e.toString());}}private static void openBrowserMac(final String url) throws Exception {- Process p = Runtime.getRuntime().exec(new String[] {"/usr/bin/osascript", "-e", "open location \"" + url + "\""});+ final Process p = Runtime.getRuntime().exec(new String[] {"/usr/bin/osascript", "-e", "open location \"" + url + "\""});p.waitFor();if (p.exitValue() != 0) {throw new RuntimeException("Mac Exec Error: " + errorResponse(p));@@ -131,12 +131,13 @@ public class Browser {* @throws Exception when an error occured*/private static void openDefaultUnixBrowser(final String url) throws Exception {- /* Use the freedesktop xdg-open to open url with the default browser.- * xdg-open is included in xdg-utils tools set (https://www.freedesktop.org/wiki/Software/xdg-utils/)- * It is part of the LSB (Linux Standard Base) and therefore included in all recent Linux Distributions supporting it- * (see https://www.linuxbase.org/navigator/browse/cmd_single.php?cmd=list-by-name&Section=ABI&Cname=xdg-open) */- String cmd = "xdg-open " + url;- Process p = Runtime.getRuntime().exec(cmd);+ /* Use the freedesktop xdg-open to open url with the default browser.+ * xdg-open is included in xdg-utils tools set (https://www.freedesktop.org/wiki/Software/xdg-utils/)+ * It is part of the LSB (Linux Standard Base) and therefore included in all recent Linux Distributions supporting it+ * (see https://www.linuxbase.org/navigator/browse/cmd_single.php?cmd=list-by-name&Section=ABI&Cname=xdg-open) */+ final String cmd = "xdg-open " + url;+ @SuppressWarnings("deprecation")+ final Process p = Runtime.getRuntime().exec(cmd);p.waitFor();if (p.exitValue() != 0) {throw new RuntimeException("Unix Exec Error/xdg-open: " + errorResponse(p));@@ -152,7 +153,8 @@ public class Browser {cmd = "rundll32 url.dll,FileProtocolHandler \"" + url + "\"";}//cmd = "cmd.exe /c start javascript:document.location='" + url + "'";- Process p = Runtime.getRuntime().exec(cmd);+ @SuppressWarnings("deprecation")+ final Process p = Runtime.getRuntime().exec(cmd);p.waitFor();if (p.exitValue() != 0) {throw new RuntimeException("EXEC ERROR: " + errorResponse(p));@@ -183,15 +185,15 @@ public class Browser {* @param args*/public static void main(final String[] args) {- try {- if (args.length > 0 && "-u".equals(args[0])) {- openBrowser(args[1]);- } else {- System.out.println("Usage java " + Browser.class.getCanonicalName() + " -u [URL]");- }- } finally {- ConcurrentLog.shutdown();- }- System.out.println("The End!");- }+ try {+ if (args.length > 0 && "-u".equals(args[0])) {+ openBrowser(args[1]);+ } else {+ System.out.println("Usage java " + Browser.class.getCanonicalName() + " -u [URL]");+ }+ } finally {+ ConcurrentLog.shutdown();+ }+ System.out.println("The End!");+ }}diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.javaindex 711d0c71a..408ee459d 100644--- a/source/net/yacy/kelondro/util/FileUtils.java+++ b/source/net/yacy/kelondro/util/FileUtils.java@@ -56,15 +56,14 @@ import java.util.Map;import java.util.concurrent.ConcurrentHashMap;import java.util.zip.GZIPInputStream;-import net.yacy.cora.document.encoding.UTF8;-import net.yacy.cora.storage.Files;-import net.yacy.cora.util.ConcurrentLog;-import org.apache.commons.lang.StringUtils;-import org.mozilla.intl.chardet.nsDetector;import org.mozilla.intl.chardet.nsPSMDetector;+import net.yacy.cora.document.encoding.UTF8;+import net.yacy.cora.storage.Files;+import net.yacy.cora.util.ConcurrentLog;+public final class FileUtils {private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size@@ -98,7 +97,7 @@ public final class FileUtils {* @see #copy(File source, File dest)*/public static long copy(final InputStream source, final OutputStream dest, final long count)- throws IOException {+ throws IOException {assert count < 0 || count > 0 : "precondition violated: count == " + count + " (nothing to copy)";if ( count == 0 ) {// no bytes to copy@@ -106,15 +105,15 @@ public final class FileUtils {}final byte[] buffer = new byte[DEFAULT_BUFFER_SIZE];- int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE);+ final int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE);int c;long total = 0;long remaining;if(count > 0) {- remaining = count;+ remaining = count;} else {- remaining = Long.MAX_VALUE;+ remaining = Long.MAX_VALUE;}while ( (c = source.read(buffer, 0, remaining < chunkSize ? (int)remaining : chunkSize)) > 0 ) {dest.write(buffer, 0, c);@@ -158,7 +157,7 @@ public final class FileUtils {* @throws NullPointerException when a parameter is null*/public static int copy(final InputStream source, final Writer dest, final Charset inputCharset)- throws IOException {+ throws IOException {final InputStreamReader reader = new InputStreamReader(source, inputCharset);return copy(reader, dest);}@@ -212,8 +211,8 @@ public final class FileUtils {// an "sun.io.MalformedInputException: Missing byte-order mark" - exception may occur here//Log.logException(e);throw new IOException(- e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(),- e);+ e == null ? "null" : e.getMessage() == null ? e.toString() : e.getMessage(),+ e);}return count;}@@ -260,8 +259,8 @@ public final class FileUtils {fos.close();} catch (final Exception e ) {ConcurrentLog.warn(- "FileUtils",- "cannot close FileOutputStream for " + dest + "! " + e.getMessage());+ "FileUtils",+ "cannot close FileOutputStream for " + dest + "! " + e.getMessage());}}}@@ -282,17 +281,17 @@ public final class FileUtils {* @see #copy(File source, File dest)*/public static void copyRange(final File source, final OutputStream dest, final int start)- throws IOException {+ throws IOException {InputStream fis = null;try {fis = new FileInputStream(source);final long skipped = fis.skip(start);if ( skipped != start ) {throw new IllegalStateException("Unable to skip '"- + start- + "' bytes. Only '"- + skipped- + "' bytes skipped.");+ + start+ + "' bytes. Only '"+ + skipped+ + "' bytes skipped.");}copy(fis, dest, -1);} finally {@@ -300,7 +299,7 @@ public final class FileUtils {try {fis.close();} catch (final Exception e ) {- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);}}}@@ -328,7 +327,7 @@ public final class FileUtils {try {fis.close();} catch (final Exception e ) {- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);}}}@@ -368,17 +367,17 @@ public final class FileUtils {* @throws NullPointerException when source parameter is null*/public static byte[] read(final InputStream source) throws IOException {- byte[] content;- try {- content = read(source, -1);- } finally {- /* source input stream must be closed here in all cases */- try {- source.close();- } catch(IOException ignoredException) {- }- }- return content;+ byte[] content;+ try {+ content = read(source, -1);+ } finally {+ /* source input stream must be closed here in all cases */+ try {+ source.close();+ } catch(final IOException ignoredException) {+ }+ }+ return content;}/**@@ -392,7 +391,7 @@ public final class FileUtils {*/public static byte[] read(final InputStream source, final int count) throws IOException {if(count == 0) {- return new byte[0];+ return new byte[0];}final ByteArrayOutputStream baos = new ByteArrayOutputStream(512);copy(source, baos, count);@@ -414,7 +413,7 @@ public final class FileUtils {try {fis.close();} catch (final Exception e ) {- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + source);}}fis = null;@@ -470,17 +469,17 @@ public final class FileUtils {return source;}- /**- * Generate a set of strings matching each line of the given file. Lines are- * lower cased and any eventual surrounding space characters are removed. Empty- * lines and lines starting with the '#' character are ignored.- *- * @param file- * a file to load- * @return a set of strings eventually empty- */+ /**+ * Generate a set of strings matching each line of the given file. Lines are+ * lower cased and any eventual surrounding space characters are removed. Empty+ * lines and lines starting with the '#' character are ignored.+ *+ * @param file+ * a file to load+ * @return a set of strings eventually empty+ */public static HashSet<String> loadList(final File file) {- final HashSet<String> set = new HashSet<String>();+ final HashSet<String> set = new HashSet<>();BufferedReader br = null;try {br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));@@ -497,7 +496,7 @@ public final class FileUtils {try {br.close();} catch (final Exception e ) {- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file);+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + file);}}}@@ -516,10 +515,10 @@ public final class FileUtils {}public static ConcurrentHashMap<String, byte[]> loadMapB(final File f) {- ConcurrentHashMap<String, String> m = loadMap(f);+ final ConcurrentHashMap<String, String> m = loadMap(f);if (m == null) return null;- ConcurrentHashMap<String, byte[]> mb = new ConcurrentHashMap<String, byte[]>();- for (Map.Entry<String, String> e: m.entrySet()) mb.put(e.getKey(), UTF8.getBytes(e.getValue()));+ final ConcurrentHashMap<String, byte[]> mb = new ConcurrentHashMap<>();+ for (final Map.Entry<String, String> e: m.entrySet()) mb.put(e.getKey(), UTF8.getBytes(e.getValue()));return mb;}@@ -529,7 +528,7 @@ public final class FileUtils {private final static String[] unescaped_strings_out = {"\\", "\n", "="};public static void saveMap(final File file, final Map<String, String> props, final String comment) {- boolean err = false;+ boolean err = false;PrintWriter pw = null;final File tf = new File(file.toString() + "." + (System.currentTimeMillis() % 1000));try {@@ -550,10 +549,7 @@ public final class FileUtils {pw.println(key + "=" + value);}pw.println("# EOF");- } catch (final FileNotFoundException e ) {- ConcurrentLog.warn("FileUtils", e.getMessage(), e);- err = true;- } catch (final UnsupportedEncodingException e ) {+ } catch (final FileNotFoundException | UnsupportedEncodingException e ) {ConcurrentLog.warn("FileUtils", e.getMessage(), e);err = true;} finally {@@ -568,10 +564,10 @@ public final class FileUtils {// ignore}}-+public static void saveMapB(final File file, final Map<String, byte[]> props, final String comment) {- HashMap<String, String> m = new HashMap<String, String>();- for (Map.Entry<String, byte[]> e: props.entrySet()) m.put(e.getKey(), UTF8.String(e.getValue()));+ final HashMap<String, String> m = new HashMap<>();+ for (final Map.Entry<String, byte[]> e: props.entrySet()) m.put(e.getKey(), UTF8.String(e.getValue()));saveMap(file, m, comment);}@@ -582,7 +578,7 @@ public final class FileUtils {public static ConcurrentHashMap<String, String> table(final Iterator<String> li) {String line;- final ConcurrentHashMap<String, String> props = new ConcurrentHashMap<String, String>();+ final ConcurrentHashMap<String, String> props = new ConcurrentHashMap<>();while ( li.hasNext() ) {int pos = 0;line = li.next().trim();@@ -594,8 +590,8 @@ public final class FileUtils {pos = line.indexOf('=', pos + 1);} while ( pos > 0 && line.charAt(pos - 1) == '\\' );if ( pos > 0 ) try {- String key = StringUtils.replaceEach(line.substring(0, pos).trim(), escaped_strings_in, unescaped_strings_out);- String value = StringUtils.replaceEach(line.substring(pos + 1).trim(), escaped_strings_in, unescaped_strings_out);+ final String key = StringUtils.replaceEach(line.substring(0, pos).trim(), escaped_strings_in, unescaped_strings_out);+ final String value = StringUtils.replaceEach(line.substring(pos + 1).trim(), escaped_strings_in, unescaped_strings_out);//System.out.println("key = " + key + ", value = " + value);props.put(key, value);} catch (final IndexOutOfBoundsException e) {@@ -606,7 +602,7 @@ public final class FileUtils {}public static Map<String, String> table(final byte[] a) {- if (a == null) return new ConcurrentHashMap<String, String>();+ if (a == null) return new ConcurrentHashMap<>();//System.out.println("***TABLE: a.size = " + a.length);return table(strings(a));}@@ -627,7 +623,7 @@ public final class FileUtils {*/public static ArrayList<String> getListArray(final File listFile) {String line;- final ArrayList<String> list = new ArrayList<String>();+ final ArrayList<String> list = new ArrayList<>();BufferedReader br = null;try {br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile), StandardCharsets.UTF_8));@@ -642,7 +638,7 @@ public final class FileUtils {try {br.close();} catch (final Exception e ) {- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);}}}@@ -711,7 +707,7 @@ public final class FileUtils {try {br.close();} catch (final Exception e ) {- ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);+ ConcurrentLog.warn("FileUtils", "Could not close input stream on file " + listFile);}}}@@ -760,7 +756,7 @@ public final class FileUtils {* @return array of file names*/public static List<String> getDirListing(final File dir, final String filter) {- final List<String> ret = new LinkedList<String>();+ final List<String> ret = new LinkedList<>();File[] fileList;if ( dir != null ) {if ( !dir.exists() ) {@@ -781,39 +777,39 @@ public final class FileUtils {public static ArrayList<File> getDirsRecursive(final File dir, final String notdir) {return getDirsRecursive(dir, notdir, true);}-+/*** @param sourceDir source directory. Must be not null.* @param notdir name of dir to exlcude. Can be null* @param fileNameFilter filter to apply on file names. Can be null.- * @return list of all files passing fileFilter under sourceDir including sub directories+ * @return list of all files passing fileFilter under sourceDir including sub directories*/public static List<File> getFilesRecursive(final File sourceDir, final String notdir, final FilenameFilter fileNameFilter) {- List<File> dirList = getDirsRecursive(sourceDir,- notdir);- dirList.add(sourceDir);- List<File> files = new ArrayList<>();- for (final File dir : dirList) {- Collections.addAll(files, dir.listFiles(fileNameFilter));- }- return files;+ final List<File> dirList = getDirsRecursive(sourceDir,+ notdir);+ dirList.add(sourceDir);+ final List<File> files = new ArrayList<>();+ for (final File dir : dirList) {+ Collections.addAll(files, dir.listFiles(fileNameFilter));+ }+ return files;}/*** Returns a List of all dirs and subdirs as File Objects Warning: untested*/private static ArrayList<File> getDirsRecursive(- final File dir,- final String notdir,- final boolean excludeDotfiles) {+ final File dir,+ final String notdir,+ final boolean excludeDotfiles) {final File[] dirList = dir.listFiles();- final ArrayList<File> resultList = new ArrayList<File>();+ final ArrayList<File> resultList = new ArrayList<>();ArrayList<File> recursive;Iterator<File> iter;for ( int i = 0; i < dirList.length; i++ ) {if ( dirList[i].isDirectory()- && (!excludeDotfiles || !dirList[i].getName().startsWith("."))- && !dirList[i].getName().equals(notdir) ) {+ && (!excludeDotfiles || !dirList[i].getName().startsWith("."))+ && !dirList[i].getName().equals(notdir) ) {resultList.add(dirList[i]);recursive = getDirsRecursive(dirList[i], notdir, excludeDotfiles);iter = recursive.iterator();@@ -927,12 +923,12 @@ public final class FileUtils {// create the temp filefinal File tempFile =- File.createTempFile(- parserClassName + "_" + ((idx > -1) ? fileName.substring(0, idx) : fileName),- (!fileExt.isEmpty()) ? "." + fileExt : fileExt);+ File.createTempFile(+ parserClassName + "_" + ((idx > -1) ? fileName.substring(0, idx) : fileName),+ (!fileExt.isEmpty()) ? "." + fileExt : fileExt);return tempFile;}-+/*** delete files and directories if a directory is not empty, delete also everything inside because* deletion sometimes fails on windows, there is also a windows exec included@@ -973,7 +969,7 @@ public final class FileUtils {break;}}- */+ */if ( path.exists() ) {path.deleteOnExit();String p = "";@@ -986,6 +982,7 @@ public final class FileUtils {// deleting files on windows sometimes does not work with javatry {final String command = "cmd /C del /F /Q \"" + p + "\"";+ @SuppressWarnings("deprecation")final Process r = Runtime.getRuntime().exec(command);if ( r == null ) {ConcurrentLog.severe("FileUtils", "cannot execute command: " + command);@@ -1002,7 +999,7 @@ public final class FileUtils {}}}-+/*** Checks if a certain file is in a given directory.* @param file the file to check@@ -1010,9 +1007,9 @@ public final class FileUtils {* @return true if file is contained in directory*/public static boolean isInDirectory(final File file, final File directory) {-+boolean inDirectory;-+try {inDirectory = (directory != null@@ -1024,10 +1021,10 @@ public final class FileUtils {} catch (final IOException e) {inDirectory = false;}-+return inDirectory;}-+/*** Auto-detect the charset of content in a stream.* Used code from http://jchardet.sourceforge.net/.@@ -1040,28 +1037,28 @@ public final class FileUtils {public static List<String> detectCharset(final InputStream inStream) throws IOException {// auto-detect charset, used code from http://jchardet.sourceforge.net/; see also: http://www-archive.mozilla.org/projects/intl/chardet.htmlList<String> result;- nsDetector det = new nsDetector(nsPSMDetector.ALL);- byte[] buf = new byte[1024] ;+ final nsDetector det = new nsDetector(nsPSMDetector.ALL);+ final byte[] buf = new byte[1024] ;int len;boolean done = false ;boolean isAscii = true ;while ((len = inStream.read(buf,0,buf.length)) != -1) {if (isAscii) {- isAscii = det.isAscii(buf,len);+ isAscii = det.isAscii(buf,len);}if (!isAscii && !done) {- done = det.DoIt(buf,len, false);+ done = det.DoIt(buf,len, false);}} det.DataEnd();result = new ArrayList<>();if (isAscii) {result.add(StandardCharsets.US_ASCII.name());} else {- for (String c: det.getProbableCharsets()) result.add(c); // worst case this returns "nomatch"+ for (final String c: det.getProbableCharsets()) result.add(c); // worst case this returns "nomatch"}return result;}-+/*** Because the checking of very large files for their charset may take some time, we do this concurrently in this method* This method does not return anything but it logs an info line if the charset is a good choice@@ -1071,19 +1068,19 @@ public final class FileUtils {* @param concurrent if this shall run concurrently*/public static void checkCharset(final File file, final String givenCharset, final boolean concurrent) {- Thread t = new Thread("FileUtils.checkCharset") {+ final Thread t = new Thread("FileUtils.checkCharset") {@Overridepublic void run() {- try (final FileInputStream fileStream = new FileInputStream(file);- final BufferedInputStream imp = new BufferedInputStream(fileStream)) { // try-with-resource to close resources- List<String> charsets = FileUtils.detectCharset(imp);+ try (final FileInputStream fileStream = new FileInputStream(file);+ final BufferedInputStream imp = new BufferedInputStream(fileStream)) { // try-with-resource to close resources+ final List<String> charsets = FileUtils.detectCharset(imp);if (charsets.contains(givenCharset)) {ConcurrentLog.info("checkCharset", "appropriate charset '" + givenCharset + "' for import of " + file + ", is part one detected " + charsets);} else {ConcurrentLog.warn("checkCharset", "possibly wrong charset '" + givenCharset + "' for import of " + file + ", use one of " + charsets);}- } catch (IOException e) {}-+ } catch (final IOException e) {}+}};if (concurrent) t.start(); else t.run();diff --git a/source/net/yacy/kelondro/util/OS.java b/source/net/yacy/kelondro/util/OS.javaindex beca081a1..abf7d1695 100644--- a/source/net/yacy/kelondro/util/OS.java+++ b/source/net/yacy/kelondro/util/OS.java@@ -41,7 +41,7 @@ import net.yacy.server.serverCore;public final class OS {- // constants for system identification+ // constants for system identificationpublic enum System {MacOSC, // 'classic' Mac OS 7.6.1/8.*/9.*MacOSX, // all Mac OS X@@ -50,125 +50,129 @@ public final class OS {Unknown; // any other system}- // constants for file type identification (Mac only)- public static final String blankTypeString = "____";-- // system-identification statics- private static final System systemOS;- public static final boolean isMacArchitecture;- private static final boolean isUnixFS;- public static final boolean canExecUnix;- public static final boolean isWindows;- public static final boolean isWin32;-- // calculated system constants- public static int maxPathLength = 65535;-- // Macintosh-specific statics- public static final Map<String, String> macFSTypeCache = new HashMap<String, String>();- public static final Map<String, String> macFSCreatorCache = new HashMap<String, String>();-- // static initialization- static {- // check operation system type- final Properties sysprop = java.lang.System.getProperties();- final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT);- if (sysname.startsWith("mac os x")) systemOS = System.MacOSX;- else if (sysname.startsWith("mac os")) systemOS = System.MacOSC;- else if (sysname.startsWith("windows")) systemOS = System.Windows;- else if ((sysname.startsWith("linux")) || (sysname.startsWith("unix"))) systemOS = System.Unix;- else systemOS = System.Unknown;-- isMacArchitecture = ((systemOS == System.MacOSC) || (systemOS == System.MacOSX));- isUnixFS = ((systemOS == System.MacOSX) || (systemOS == System.Unix));- canExecUnix = ((isUnixFS) || (!((systemOS == System.MacOSC) || (systemOS == System.Windows))));- isWindows = (systemOS == System.Windows);- isWin32 = (isWindows && java.lang.System.getProperty("os.arch", "").contains("x86"));-- // set up maximum path length according to system- if (isWindows) maxPathLength = 255; else maxPathLength = 65535;- }-- public static String infoString() {- String s = "System=";- if (systemOS == System.Unknown) s += "unknown";- else if (systemOS == System.MacOSC) s += "Mac OS Classic";- else if (systemOS == System.MacOSX) s += "Mac OS X";- else if (systemOS == System.Unix) s += "Unix/Linux";- else if (systemOS == System.Windows) s += "Windows";- else s += "unknown";- if (isMacArchitecture) s += ", Mac System Architecture";- if (isUnixFS) s += ", has Unix-like File System";- if (canExecUnix) s += ", can execute Unix-Shell Commands";- return s;- }-- /** generates a 2-character string containing information about the OS-type*/- public static String infoKey() {- String s = "";- if (systemOS == System.Unknown) s += "o";- else if (systemOS == System.MacOSC) s += "c";- else if (systemOS == System.MacOSX) s += "x";- else if (systemOS == System.Unix) s += "u";- else if (systemOS == System.Windows) s += "w";- else s += "o";- if (isMacArchitecture) s += "m";- if (isUnixFS) s += "f";- if (canExecUnix) s += "e";- return s;- }-- public static void deployScript(final File scriptFile, final String theScript) throws IOException {- FileUtils.copy(UTF8.getBytes(theScript), scriptFile);- if(!isWindows){ // set executable- try {- Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();- } catch (final InterruptedException e) {- ConcurrentLog.severe("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e);- throw new IOException(e.getMessage());- }- }- }+ // constants for file type identification (Mac only)+ public static final String blankTypeString = "____";++ // system-identification statics+ private static final System systemOS;+ public static final boolean isMacArchitecture;+ private static final boolean isUnixFS;+ public static final boolean canExecUnix;+ public static final boolean isWindows;+ public static final boolean isWin32;++ // calculated system constants+ public static int maxPathLength = 65535;++ // Macintosh-specific statics+ public static final Map<String, String> macFSTypeCache = new HashMap<>();+ public static final Map<String, String> macFSCreatorCache = new HashMap<>();++ // static initialization+ static {+ // check operation system type+ final Properties sysprop = java.lang.System.getProperties();+ final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT);+ if (sysname.startsWith("mac os x")) systemOS = System.MacOSX;+ else if (sysname.startsWith("mac os")) systemOS = System.MacOSC;+ else if (sysname.startsWith("windows")) systemOS = System.Windows;+ else if ((sysname.startsWith("linux")) || (sysname.startsWith("unix"))) systemOS = System.Unix;+ else systemOS = System.Unknown;++ isMacArchitecture = ((systemOS == System.MacOSC) || (systemOS == System.MacOSX));+ isUnixFS = ((systemOS == System.MacOSX) || (systemOS == System.Unix));+ canExecUnix = ((isUnixFS) || (!((systemOS == System.MacOSC) || (systemOS == System.Windows))));+ isWindows = (systemOS == System.Windows);+ isWin32 = (isWindows && java.lang.System.getProperty("os.arch", "").contains("x86"));++ // set up maximum path length according to system+ if (isWindows) maxPathLength = 255; else maxPathLength = 65535;+ }++ public static String infoString() {+ String s = "System=";+ if (systemOS == System.Unknown) s += "unknown";+ else if (systemOS == System.MacOSC) s += "Mac OS Classic";+ else if (systemOS == System.MacOSX) s += "Mac OS X";+ else if (systemOS == System.Unix) s += "Unix/Linux";+ else if (systemOS == System.Windows) s += "Windows";+ else s += "unknown";+ if (isMacArchitecture) s += ", Mac System Architecture";+ if (isUnixFS) s += ", has Unix-like File System";+ if (canExecUnix) s += ", can execute Unix-Shell Commands";+ return s;+ }++ /** generates a 2-character string containing information about the OS-type*/+ public static String infoKey() {+ String s = "";+ if (systemOS == System.Unknown) s += "o";+ else if (systemOS == System.MacOSC) s += "c";+ else if (systemOS == System.MacOSX) s += "x";+ else if (systemOS == System.Unix) s += "u";+ else if (systemOS == System.Windows) s += "w";+ else s += "o";+ if (isMacArchitecture) s += "m";+ if (isUnixFS) s += "f";+ if (canExecUnix) s += "e";+ return s;+ }- /**- * use a hack to get the current process PID- * @return the PID of the current java process or -1 if the PID cannot be obtained- */- public static int getPID() {+ @SuppressWarnings("deprecation")+ public static void deployScript(final File scriptFile, final String theScript) throws IOException {+ FileUtils.copy(UTF8.getBytes(theScript), scriptFile);+ if(!isWindows){ // set executable+ try {+ Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();+ } catch (final InterruptedException e) {+ ConcurrentLog.severe("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e);+ throw new IOException(e.getMessage());+ }+ }+ }++ /**+ * use a hack to get the current process PID+ * @return the PID of the current java process or -1 if the PID cannot be obtained+ */+ public static int getPID() {final String pids = ManagementFactory.getRuntimeMXBean().getName();final int p = pids.indexOf('@');return p >= 0 ? NumberTools.parseIntDecSubstring(pids, 0, p) : -1;- }-- public static void execAsynchronous(final File scriptFile) throws IOException {- // runs a script as separate thread- String starterFileExtension = null;- String script = null;- if(isWindows){- starterFileExtension = ".starter.bat";- // use /K to debug, /C for release- script = "start /MIN CMD /C \"" + scriptFile.getAbsolutePath() + "\"";- } else { // unix/linux- starterFileExtension = ".starter.sh";- script = "#!/bin/sh" + serverCore.LF_STRING + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + " &" + serverCore.LF_STRING;- }- final File starterFile = new File(scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + starterFileExtension);- deployScript(starterFile, script);- try {- Runtime.getRuntime().exec(starterFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();- } catch (final InterruptedException e) {- throw new IOException(e.getMessage());- }- FileUtils.deletedelete(starterFile);- }+ }+ @SuppressWarnings("deprecation")+ public static void execAsynchronous(final File scriptFile) throws IOException {+ // runs a script as separate thread+ String starterFileExtension = null;+ String script = null;+ if(isWindows){+ starterFileExtension = ".starter.bat";+ // use /K to debug, /C for release+ script = "start /MIN CMD /C \"" + scriptFile.getAbsolutePath() + "\"";+ } else { // unix/linux+ starterFileExtension = ".starter.sh";+ script = "#!/bin/sh" + serverCore.LF_STRING + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + " &" + serverCore.LF_STRING;+ }+ final File starterFile = new File(scriptFile.getAbsolutePath().replaceAll(" ", "\\ ") + starterFileExtension);+ deployScript(starterFile, script);+ try {+ Runtime.getRuntime().exec(starterFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();+ } catch (final InterruptedException e) {+ throw new IOException(e.getMessage());+ }+ FileUtils.deletedelete(starterFile);+ }++ @SuppressWarnings("deprecation")public static List<String> execSynchronous(final String command) throws IOException {// runs a unix/linux command and returns output as Vector of Strings// this method blocks until the command is executedfinal Process p = Runtime.getRuntime().exec(command);return readStreams(p);}-++ @SuppressWarnings("deprecation")public static List<String> execSynchronous(final String[] command) throws IOException {// runs a unix/linux command and returns output as Vector of Strings// this method blocks until the command is executed@@ -184,40 +188,40 @@ public final class OS {*/public static List<String> readStreams(final Process p) throws IOException {String line;- final List<String> output = new ArrayList<>();-- if(p == null) {- return output;- }-- try (final InputStreamReader streamReader = new InputStreamReader(p.getInputStream());- final BufferedReader in = new BufferedReader(streamReader);) {- while ((line = in.readLine()) != null) {- output.add(line);- }- }-- try (final InputStreamReader streamReader = new InputStreamReader(p.getErrorStream());- final BufferedReader in = new BufferedReader(streamReader);) {- while ((line = in.readLine()) != null) {- output.add(line);- }- }+ final List<String> output = new ArrayList<>();++ if(p == null) {+ return output;+ }++ try (final InputStreamReader streamReader = new InputStreamReader(p.getInputStream());+ final BufferedReader in = new BufferedReader(streamReader);) {+ while ((line = in.readLine()) != null) {+ output.add(line);+ }+ }++ try (final InputStreamReader streamReader = new InputStreamReader(p.getErrorStream());+ final BufferedReader in = new BufferedReader(streamReader);) {+ while ((line = in.readLine()) != null) {+ output.add(line);+ }+ }return output;}- public static void main(final String[] args) {+ public static void main(final String[] args) {try {- List<String> v = execSynchronous("/usr/local/bin/wkhtmltoimage");- for (String r: v) java.lang.System.out.println(r);- } catch (IOException e) {+ final List<String> v = execSynchronous("/usr/local/bin/wkhtmltoimage");+ for (final String r: v) java.lang.System.out.println(r);+ } catch (final IOException e) {}/*if (args[0].equals("-m")) {java.lang.System.out.println("Maximum possible memory: " + Integer.toString(getWin32MaxHeap()) + "m");}- */- }+ */+ }}diff --git a/source/net/yacy/server/http/HTTPDProxyHandler.java b/source/net/yacy/server/http/HTTPDProxyHandler.javaindex 744c4515c..e300ec9e2 100644--- a/source/net/yacy/server/http/HTTPDProxyHandler.java+++ b/source/net/yacy/server/http/HTTPDProxyHandler.java@@ -37,7 +37,7 @@support gzip-ed encoding. We also do not support unrealistic'expires' values that would force a cache to be flushed immediatelypragma non-cache attributes are supported-*/+ */package net.yacy.server.http;@@ -67,6 +67,7 @@ import java.util.logging.FileHandler;import java.util.logging.Level;import java.util.logging.LogManager;import java.util.logging.Logger;+import javax.servlet.http.HttpServletRequest;import javax.servlet.http.HttpServletResponse;@@ -89,6 +90,7 @@ import net.yacy.repository.Blacklist.BlacklistType;import net.yacy.search.Switchboard;import net.yacy.server.serverObjects;+@SuppressWarnings("deprecation")public final class HTTPDProxyHandler {@@ -108,7 +110,7 @@ public final class HTTPDProxyHandler {// creating a loggerprivate static final ConcurrentLog log = new ConcurrentLog("PROXY");- /**+ /*** Do logging configuration for special proxy access log file*/static {@@ -117,77 +119,77 @@ public final class HTTPDProxyHandler {sb = Switchboard.getSwitchboard();if (sb != null) {- // set timeout- timeout = sb.getConfigInt("proxy.clientTimeout", 60000);+ // set timeout+ timeout = sb.getConfigInt("proxy.clientTimeout", 60000);- // do logger initialization- try {- log.info("Configuring proxy access logging ...");-- // getting the logging manager- final LogManager manager = LogManager.getLogManager();- final String className = HTTPDProxyHandler.class.getName();-- // determining if proxy access logging is enabled- final String enabled = manager.getProperty(className + ".logging.enabled");- if ("true".equalsIgnoreCase(enabled)) {-- // reading out some needed configuration properties- int limit = 1024*1024, count = 20;- String pattern = manager.getProperty(className + ".logging.FileHandler.pattern");- if (pattern == null) pattern = "DATA/LOG/proxyAccess%u%g.log";- // make pattern absolute- if (!new File(pattern).isAbsolute()) pattern = new File(sb.getDataPath(), pattern).getAbsolutePath();-- final String limitStr = manager.getProperty(className + ".logging.FileHandler.limit");- if (limitStr != null) try { limit = Integer.parseInt(limitStr); } catch (final NumberFormatException e) {}-- final String countStr = manager.getProperty(className + ".logging.FileHandler.count");- if (countStr != null) try { count = Integer.parseInt(countStr); } catch (final NumberFormatException e) {}-- // creating the proxy access logger- final Logger proxyLogger = Logger.getLogger("PROXY.access");- proxyLogger.setUseParentHandlers(false);- proxyLogger.setLevel(Level.FINEST);-- final FileHandler txtLog = new FileHandler(pattern, limit, count, true);- txtLog.setFormatter(new ProxyLogFormatter());- txtLog.setLevel(Level.FINEST);- proxyLogger.addHandler(txtLog);-- log.info("Proxy access logging configuration done." +- "\n\tFilename: " + pattern +- "\n\tLimit: " + limitStr +- "\n\tCount: " + countStr);- } else {- log.info("Proxy access logging is deactivated.");+ // do logger initialization+ try {+ log.info("Configuring proxy access logging ...");++ // getting the logging manager+ final LogManager manager = LogManager.getLogManager();+ final String className = HTTPDProxyHandler.class.getName();++ // determining if proxy access logging is enabled+ final String enabled = manager.getProperty(className + ".logging.enabled");+ if ("true".equalsIgnoreCase(enabled)) {++ // reading out some needed configuration properties+ int limit = 1024*1024, count = 20;+ String pattern = manager.getProperty(className + ".logging.FileHandler.pattern");+ if (pattern == null) pattern = "DATA/LOG/proxyAccess%u%g.log";+ // make pattern absolute+ if (!new File(pattern).isAbsolute()) pattern = new File(sb.getDataPath(), pattern).getAbsolutePath();++ final String limitStr = manager.getProperty(className + ".logging.FileHandler.limit");+ if (limitStr != null) try { limit = Integer.parseInt(limitStr); } catch (final NumberFormatException e) {}++ final String countStr = manager.getProperty(className + ".logging.FileHandler.count");+ if (countStr != null) try { count = Integer.parseInt(countStr); } catch (final NumberFormatException e) {}++ // creating the proxy access logger+ final Logger proxyLogger = Logger.getLogger("PROXY.access");+ proxyLogger.setUseParentHandlers(false);+ proxyLogger.setLevel(Level.FINEST);++ final FileHandler txtLog = new FileHandler(pattern, limit, count, true);+ txtLog.setFormatter(new ProxyLogFormatter());+ txtLog.setLevel(Level.FINEST);+ proxyLogger.addHandler(txtLog);++ log.info("Proxy access logging configuration done." ++ "\n\tFilename: " + pattern ++ "\n\tLimit: " + limitStr ++ "\n\tCount: " + countStr);+ } else {+ log.info("Proxy access logging is deactivated.");+ }+ } catch (final Exception e) {+ log.severe("Unable to configure proxy access logging.",e);}- } catch (final Exception e) {- log.severe("Unable to configure proxy access logging.",e);- }- // load the yellow-list- final String f = sb.getConfig("proxyYellowList", null);- if (f != null) {- yellowList = FileUtils.loadList(new File(f));- log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");- } else {- yellowList = null;- }+ // load the yellow-list+ final String f = sb.getConfig("proxyYellowList", null);+ if (f != null) {+ yellowList = FileUtils.loadList(new File(f));+ log.config("loaded yellow-list from file " + f + ", " + yellowList.size() + " entries");+ } else {+ yellowList = null;+ }- final String redirectorPath = sb.getConfig("externalRedirector", "");- if (redirectorPath.length() > 0 && !redirectorEnabled) {- try {- redirectorProcess=Runtime.getRuntime().exec(redirectorPath, null, null);- redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream());- redirectorReader = new BufferedReader(new InputStreamReader(redirectorProcess.getInputStream()));- redirectorEnabled=true;- } catch (final IOException e) {- System.out.println("redirector not Found");+ final String redirectorPath = sb.getConfig("externalRedirector", "");+ if (redirectorPath.length() > 0 && !redirectorEnabled) {+ try {+ redirectorProcess=Runtime.getRuntime().exec(redirectorPath, null, null);+ redirectorWriter = new PrintWriter(redirectorProcess.getOutputStream());+ redirectorReader = new BufferedReader(new InputStreamReader(redirectorProcess.getInputStream()));+ redirectorEnabled=true;+ } catch (final IOException e) {+ System.out.println("redirector not Found");+ }}- }} else {- yellowList = null;+ yellowList = null;}}@@ -342,25 +344,25 @@ public final class HTTPDProxyHandler {// case 1 and case 3if (cachedResponseHeader == null) {if (log.isFinest()) log.finest(reqID + " page not in cache: fulfill request from web");- fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent);+ fulfillRequestFromWeb(conProp, url, requestHeader, cachedResponseHeader, countedRespond, agent);} else {- final Request request = new Request(- null,+ final Request request = new Request(+ null,url,requestHeader.referer() == null ? null : requestHeader.referer().hash(),- "",- cachedResponseHeader.lastModified(),- sb.crawler.defaultProxyProfile.handle(),- 0,- sb.crawler.defaultProxyProfile.timezoneOffset());+ "",+ cachedResponseHeader.lastModified(),+ sb.crawler.defaultProxyProfile.handle(),+ 0,+ sb.crawler.defaultProxyProfile.timezoneOffset());final Response response = new Response(- request,+ request,requestHeader,cachedResponseHeader,sb.crawler.defaultProxyProfile,true,null- );+ );final byte[] cacheContent = Cache.getContent(url.hash());if (cacheContent != null && response.isFreshForProxy()) {if (log.isFinest()) log.finest(reqID + " fulfill request from cache");@@ -433,9 +435,9 @@ public final class HTTPDProxyHandler {// remove yacy-subdomain-path, when accessing /envif ((yAddress != null)- && (remotePath.startsWith("/env"))- && (yAddress.indexOf('/') != -1)- ) yAddress = yAddress.substring(0, yAddress.indexOf('/'));+ && (remotePath.startsWith("/env"))+ && (yAddress.indexOf('/') != -1)+ ) yAddress = yAddress.substring(0, yAddress.indexOf('/'));modifyProxyHeaders(requestHeader, httpVer);@@ -448,17 +450,17 @@ public final class HTTPDProxyHandler {try (final HTTPClient client = new HTTPClient(agent, timeout)) {client.setHeader(requestHeader.entrySet());client.setRedirecting(false);- client.GET(getUrl, false);+ client.GET(getUrl, false);if (log.isFinest()) log.finest(reqID +" response status: "+ client.getHttpResponse().getStatusLine());- int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();+ final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());// determine if it's an internal error of the httpcif (responseHeader.isEmpty()) {- throw new Exception(client.getHttpResponse().getStatusLine().toString());+ throw new Exception(client.getHttpResponse().getStatusLine().toString());}- ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, statusCode, respond);+ final ChunkedOutputStream chunkedOut = setTransferEncoding(conProp, responseHeader, statusCode, respond);// the cache does either not exist or is (supposed to be) stalelong sizeBeforeDelete = -1;@@ -475,19 +477,19 @@ public final class HTTPDProxyHandler {// reserver cache entryfinal Request request = new Request(- null,+ null,url,requestHeader.referer() == null ? null : requestHeader.referer().hash(),- "",- responseHeader.lastModified(),- sb.crawler.defaultProxyProfile.handle(),- 0,- sb.crawler.defaultProxyProfile.timezoneOffset());+ "",+ responseHeader.lastModified(),+ sb.crawler.defaultProxyProfile.handle(),+ 0,+ sb.crawler.defaultProxyProfile.timezoneOffset());// handle incoming cookieshandleIncomingCookies(responseHeader, host, ip);-// prepareResponseHeader(responseHeader, res.getHttpVer());+ // prepareResponseHeader(responseHeader, res.getHttpVer());prepareResponseHeader(responseHeader, client.getHttpResponse().getProtocolVersion().toString());// sending the respond header back to the client@@ -506,7 +508,7 @@ public final class HTTPDProxyHandler {if (hasBody(client.getHttpResponse().getStatusLine().getStatusCode())) {- OutputStream outStream = chunkedOut != null ? chunkedOut : respond;+ final OutputStream outStream = chunkedOut != null ? chunkedOut : respond;final Response response = new Response(request,requestHeader,@@ -514,7 +516,7 @@ public final class HTTPDProxyHandler {sb.crawler.defaultProxyProfile,true,null- );+ );final String storeError = response.shallStoreCacheForProxy();final boolean storeHTCache = response.profile().storeHTCache();final String supportError = TextParser.supports(response.url(), response.getMimeType());@@ -530,14 +532,14 @@ public final class HTTPDProxyHandler {* c) the content should be indexed*/((storeHTCache) || (supportError != null))- ) {+ ) {// we don't write actually into a file, only to RAM, and schedule writing the file.-// int l = res.getResponseHeader().size();- final int l = responseHeader.size();+ // int l = res.getResponseHeader().size();+ final int l = responseHeader.size();final ByteArrayOutputStream byteStream = new ByteArrayOutputStream((l < 32) ? 32 : l);final OutputStream toClientAndMemory = new MultiOutputStream(new OutputStream[] {outStream, byteStream});-// FileUtils.copy(res.getDataAsStream(), toClientAndMemory);+ // FileUtils.copy(res.getDataAsStream(), toClientAndMemory);client.writeTo(toClientAndMemory);// cached bytesbyte[] cacheArray;@@ -627,7 +629,7 @@ public final class HTTPDProxyHandler {final ResponseHeader cachedResponseHeader,final byte[] cacheEntry,OutputStream respond- ) throws IOException {+ ) throws IOException {final String httpVer = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HTTP_VER);final String clienthttpVer; // the http version of the client connection@@ -721,12 +723,12 @@ public final class HTTPDProxyHandler {requestHeader.put(HeaderFramework.USER_AGENT, generateUserAgent(requestHeader));}- // only gzip-encoding is supported, remove other encodings (e. g. deflate)+ // only gzip-encoding is supported, remove other encodings (e. g. deflate)if ((requestHeader.get(HeaderFramework.ACCEPT_ENCODING,"")).indexOf(HeaderFramework.CONTENT_ENCODING_GZIP,0) != -1) {requestHeader.put(HeaderFramework.ACCEPT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP);- } else {+ } else {requestHeader.put(HeaderFramework.ACCEPT_ENCODING, "");- }+ }addXForwardedForHeader(conProp, requestHeader);}@@ -782,7 +784,7 @@ public final class HTTPDProxyHandler {if (httpVer.equals(HeaderFramework.HTTP_VERSION_0_9) || httpVer.equals(HeaderFramework.HTTP_VERSION_1_0)) {forceConnectionClose(conProp);} else {- chunkedOut = new ChunkedOutputStream(respond);+ chunkedOut = new ChunkedOutputStream(respond);}responseHeader.remove(HeaderFramework.CONTENT_LENGTH);}@@ -933,12 +935,12 @@ public final class HTTPDProxyHandler {errorMessage = "IP address of the destination host could not be determined";}} else if ((exceptionMsg != null) &&- (- (exceptionMsg.indexOf("socket write error",0)>=0) ||- (exceptionMsg.indexOf("Read timed out",0) >= 0) ||- (exceptionMsg.indexOf("Broken pipe",0) >= 0) ||- (exceptionMsg.indexOf("server has closed connection",0) >= 0)- )) {+ (+ (exceptionMsg.indexOf("socket write error",0)>=0) ||+ (exceptionMsg.indexOf("Read timed out",0) >= 0) ||+ (exceptionMsg.indexOf("Broken pipe",0) >= 0) ||+ (exceptionMsg.indexOf("server has closed connection",0) >= 0)+ )) {errorMessage = exceptionMsg;ConcurrentLog.logException(e);} else {@@ -983,7 +985,7 @@ public final class HTTPDProxyHandler {final serverObjects detailedErrorMsgMap = new serverObjects();// generic toplevel domains- final HashSet<String> topLevelDomains = new HashSet<String>(Arrays.asList(new String[]{+ final HashSet<String> topLevelDomains = new HashSet<>(Arrays.asList(new String[]{"aero", // Fluggesellschaften/Luftfahrt"arpa", // Einrichtung des ARPANet"biz", // Business@@ -1012,8 +1014,8 @@ public final class HTTPDProxyHandler {}));// getting some connection properties- DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);- int orgHostPort = orgurl.getPort();+ final DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);+ final int orgHostPort = orgurl.getPort();String orgHostName = orgurl.getHost();if (orgHostName == null) orgHostName = "unknown";orgHostName = orgHostName.toLowerCase(Locale.ROOT);@@ -1025,7 +1027,7 @@ public final class HTTPDProxyHandler {detailedErrorMsgMap.put("hostName", orgHostName);// guessing hostnames- final HashSet<String> testHostNames = new HashSet<String>();+ final HashSet<String> testHostNames = new HashSet<>();String testHostName = null;if (!orgHostName.startsWith("www.")) {testHostName = "www." + orgHostName;@@ -1042,7 +1044,7 @@ public final class HTTPDProxyHandler {if (addr != null) testHostNames.add(testHostName);}- int pos = orgHostName.lastIndexOf('.');+ final int pos = orgHostName.lastIndexOf('.');if (pos != -1) {final Iterator<String> iter = topLevelDomains.iterator();while (iter.hasNext()) {@@ -1142,7 +1144,7 @@ public final class HTTPDProxyHandler {logMessage.append(' ');// Method- HttpServletRequest origrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);+ final HttpServletRequest origrequest = (HttpServletRequest) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST);final String requestMethod = origrequest.getMethod();logMessage.append(requestMethod);logMessage.append(' ');diff --git a/source/net/yacy/visualization/DemoApplet.java b/source/net/yacy/visualization/DemoApplet.javadeleted file mode 100644index 55ceb5430..000000000--- a/source/net/yacy/visualization/DemoApplet.java+++ /dev/null@@ -1,75 +0,0 @@-package net.yacy.visualization;--import java.applet.Applet;-import java.awt.Dimension;-import java.awt.Graphics;----@SuppressWarnings("removal")-public class DemoApplet extends Applet implements Runnable {- // can be run in eclipse with- // Run -> Run As -> Java Applet-- // see http://www.javaworld.com/javaworld/jw-03-1996/jw-03-animation.html?page=3-- private static final long serialVersionUID = -8230253094143014406L;-- private int delay;- private Thread animator;- private RasterPlotter offGraphics;-- @Override- public void init() {- final String str = getParameter("fps");- final int fps = (str != null) ? Integer.parseInt(str) : 10;- delay = (fps > 0) ? (1000 / fps) : 100;- }-- @Override- public void start() {- animator = new Thread(this);- animator.start();- }-- @Override- public void run() {- while (Thread.currentThread() == animator) {- final long time = System.currentTimeMillis();- repaint();- try {- Thread.sleep(delay - System.currentTimeMillis() + time);- } catch (final InterruptedException e) {- break;- }- }- }-- @Override- public void stop() {- animator = null;- }-- @Override- public void update(final Graphics g) {- final Dimension d = getSize();- offGraphics = new RasterPlotter(d.width, d.height, RasterPlotter.DrawMode.MODE_REPLACE, "FFFFFF");- paintFrame(offGraphics);- g.drawImage(offGraphics.getImage(), 0, 0, null);- }-- @Override- public void paint(final Graphics g) {- if (offGraphics != null) {- g.drawImage(offGraphics.getImage(), 0, 0, null);- }- }-- public void paintFrame(final RasterPlotter m) {- RasterPlotter.demoPaint(m);- final int y = (int) (System.currentTimeMillis() / 10 % 300);- m.setColor(RasterPlotter.GREY);- PrintTool.print(m, 0, y, 0, "Hello World", -1, 100);- }--}