Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
moved all remaining servlet classes to new location
Michael Peter Christenadbda4c71b38
diff --git a/htroot/AccessGrid_p.java b/source/net/yacy/htroot/AccessGrid_p.javasimilarity index 98%rename from htroot/AccessGrid_p.javarename to source/net/yacy/htroot/AccessGrid_p.javaindex 09cffd1c2..3174084b9 100644--- a/htroot/AccessGrid_p.java+++ b/source/net/yacy/htroot/AccessGrid_p.java@@ -22,6 +22,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;diff --git a/htroot/AccessPicture_p.java b/source/net/yacy/htroot/AccessPicture_p.javasimilarity index 97%rename from htroot/AccessPicture_p.javarename to source/net/yacy/htroot/AccessPicture_p.javaindex e1432fc5f..782815be4 100644--- a/htroot/AccessPicture_p.java+++ b/source/net/yacy/htroot/AccessPicture_p.java@@ -22,6 +22,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.util.ConcurrentModificationException;import java.util.Date;import java.util.Iterator;@@ -89,10 +91,10 @@ public class AccessPicture_p {// draw home peerfinal int centerx = (picture.gridWidth() >> 1) - 1;final int centery = picture.gridHeight() >> 1;- long color_dot_l = Long.parseLong(color_dot, 16);- long color_text_l = Long.parseLong(color_text, 16);- long color_line_l = Long.parseLong(color_line, 16);- long color_grid_l = Long.parseLong(color_grid, 16);+ final long color_dot_l = Long.parseLong(color_dot, 16);+ final long color_text_l = Long.parseLong(color_text, 16);+ final long color_line_l = Long.parseLong(color_line, 16);+ final long color_grid_l = Long.parseLong(color_grid, 16);picture.setColor(color_dot_l);picture.gridDot(centerx, centery, 5, true, 100);if (corona) {diff --git a/htroot/AccessTracker_p.java b/source/net/yacy/htroot/AccessTracker_p.javasimilarity index 94%rename from htroot/AccessTracker_p.javarename to source/net/yacy/htroot/AccessTracker_p.javaindex 07f9e422d..efceebc1a 100644--- a/htroot/AccessTracker_p.java+++ b/source/net/yacy/htroot/AccessTracker_p.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Collection;import java.util.ConcurrentModificationException;import java.util.Date;@@ -46,9 +48,9 @@ import net.yacy.search.Switchboard;import net.yacy.search.query.AccessTracker;import net.yacy.search.query.QueryParams;import net.yacy.server.serverAccessTracker;+import net.yacy.server.serverAccessTracker.Track;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;-import net.yacy.server.serverAccessTracker.Track;public class AccessTracker_p {@@ -59,7 +61,7 @@ public class AccessTracker_p {} catch (final ConcurrentModificationException e) {}return accessClone;}-+/*** Remove any eventual slash character at the beginning of path.* @param path a path relative to YaCy context@@ -125,7 +127,7 @@ public class AccessTracker_p {prop.putHTML("page_list_" + entCount + "_host", host);prop.put("page_list_" + entCount + "_date", GenericFormatter.SIMPLE_FORMATTER.format(new Date(entry.getTime())));prop.putHTML("page_list_" + entCount + "_path", entry.getPath());- /* For better integration of YaCy peers behind a reverse proxy subfolder,+ /* For better integration of YaCy peers behind a reverse proxy subfolder,* ensure a path relative to this servlet (with no starting slash) is used for links URLs.* We keep the pathes starting with a slash for display only. */prop.putHTML("page_list_" + entCount + "_relativePath", getRelativePathToThis(entry.getPath()));@@ -148,7 +150,7 @@ public class AccessTracker_p {prop.putHTML("page_list_" + entCount + "_host", host);prop.put("page_list_" + entCount + "_date", GenericFormatter.SIMPLE_FORMATTER.format(new Date(entry.getTime())));prop.putHTML("page_list_" + entCount + "_path", entry.getPath());- /* For better integration of YaCy peers behind a reverse proxy subfolder,+ /* For better integration of YaCy peers behind a reverse proxy subfolder,* ensure a path relative to this servlet (with no starting slash) is used for links URLs.* We keep the pathes starting with a slash for display only. */prop.putHTML("page_list_" + entCount + "_relativePath", getRelativePathToThis(entry.getPath()));@@ -172,8 +174,8 @@ public class AccessTracker_p {long utimeSum = 0;long stimeSum = 0;long rtimeSum = 0;- long utimeSum1 = 0;- long stimeSum1 = 0;+ final long utimeSum1 = 0;+ final long stimeSum1 = 0;long rtimeSum1 = 0;int m = 0;@@ -213,13 +215,13 @@ public class AccessTracker_p {utimeSum += query.urlretrievaltime;stimeSum += query.snippetcomputationtime;rtimeSum += query.searchtime;-+if (query.transmitcount > 0){rcount++;/* utimeSum1 += query.urlretrievaltime;stimeSum1 += query.snippetcomputationtime; */rtimeSum1 += query.searchtime;- }+ }m++;}prop.put("page_list", m);@@ -267,15 +269,15 @@ public class AccessTracker_p {// calculate top search query wordsif (page == 2 && AccessTracker.getDumpFile() != null && AccessTracker.getDumpFile().exists()) {- Date toDate = new Date();- Date fromDate = new Date(toDate.getTime() - 7 * 24 * 3600 * 1000); // 7 Days earlier- List<EventTracker.Event> evList = AccessTracker.readLog(AccessTracker.getDumpFile(), fromDate, toDate);- OrderedScoreMap<String> topicNavigator = new OrderedScoreMap<String>(String.CASE_INSENSITIVE_ORDER);- for (EventTracker.Event ev : evList) {- String qs = ev.payload.toString();+ final Date toDate = new Date();+ final Date fromDate = new Date(toDate.getTime() - 7 * 24 * 3600 * 1000); // 7 Days earlier+ final List<EventTracker.Event> evList = AccessTracker.readLog(AccessTracker.getDumpFile(), fromDate, toDate);+ final OrderedScoreMap<String> topicNavigator = new OrderedScoreMap<String>(String.CASE_INSENSITIVE_ORDER);+ for (final EventTracker.Event ev : evList) {+ final String qs = ev.payload.toString();if (qs.startsWith("qs ")) { // currently only raw querystring "qs" lines are included- String[] words = CommonPattern.SPACE.split(qs.substring(3));- for (String w : words) {+ final String[] words = CommonPattern.SPACE.split(qs.substring(3));+ for (final String w : words) {if (!w.isEmpty()) {topicNavigator.inc(w);}@@ -294,8 +296,8 @@ public class AccessTracker_p {prop.put("page_nav-topics", "1");String name;int i = 0;- int maxcount = topicNavigator.getMaxScore();- Iterator<String> navigatorIterator = topicNavigator.iterator();+ final int maxcount = topicNavigator.getMaxScore();+ final Iterator<String> navigatorIterator = topicNavigator.iterator();while (i < TOPWORDS_MAXCOUNT && navigatorIterator.hasNext()) {name = navigatorIterator.next();diff --git a/htroot/Autocrawl_p.java b/source/net/yacy/htroot/Autocrawl_p.javasimilarity index 98%rename from htroot/Autocrawl_p.javarename to source/net/yacy/htroot/Autocrawl_p.javaindex 27c629a2d..02122f62b 100644--- a/htroot/Autocrawl_p.java+++ b/source/net/yacy/htroot/Autocrawl_p.java@@ -1,3 +1,5 @@+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.search.Switchboard;import net.yacy.search.SwitchboardConstants;@@ -9,7 +11,7 @@ public class Autocrawl_p {public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {final serverObjects prop = new serverObjects();final Switchboard sb = (Switchboard) env;-+boolean autocrawlEnable = sb.getConfigBool(SwitchboardConstants.AUTOCRAWL, false);int autocrawlRatio = Integer.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_RATIO, "50"));int autocrawlRows = Integer.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_ROWS, "100"));@@ -19,7 +21,7 @@ public class Autocrawl_p {int autocrawlDeep = Integer.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_DEEP_DEPTH, "3"));boolean autocrawlText = sb.getConfigBool(SwitchboardConstants.AUTOCRAWL_INDEX_TEXT, true);boolean autocrawlMedia = sb.getConfigBool(SwitchboardConstants.AUTOCRAWL_INDEX_MEDIA, true);-+if (post != null) {autocrawlEnable = post.getBoolean("autocrawlEnable");if (post.containsKey("autocrawlRatio")) {@@ -43,7 +45,7 @@ public class Autocrawl_p {autocrawlText = post.getBoolean("autocrawlText");autocrawlMedia = post.getBoolean("autocrawlMedia");}-+if (autocrawlRatio > 500) {autocrawlRatio = 500;} else if (autocrawlRatio < 1) {@@ -69,7 +71,7 @@ public class Autocrawl_p {} else if (autocrawlDeep < 1) {autocrawlDeep = 1;}-+if (post != null) {sb.setConfig(SwitchboardConstants.AUTOCRAWL, autocrawlEnable);sb.setConfig(SwitchboardConstants.AUTOCRAWL_RATIO, autocrawlRatio);@@ -80,12 +82,12 @@ public class Autocrawl_p {sb.setConfig(SwitchboardConstants.AUTOCRAWL_DEEP_DEPTH, autocrawlDeep);sb.setConfig(SwitchboardConstants.AUTOCRAWL_INDEX_TEXT, autocrawlText);sb.setConfig(SwitchboardConstants.AUTOCRAWL_INDEX_MEDIA, autocrawlMedia);-+sb.initAutocrawl(autocrawlEnable);-+prop.put("changed", true);}-+prop.put("autocrawlEnable", autocrawlEnable);prop.put("autocrawlRatio", autocrawlRatio);prop.put("autocrawlRows", autocrawlRows);@@ -95,7 +97,7 @@ public class Autocrawl_p {prop.put("autocrawlDeep", autocrawlDeep);prop.put("autocrawlText", autocrawlText);prop.put("autocrawlMedia", autocrawlMedia);-+return prop;}}diff --git a/htroot/Banner.java b/source/net/yacy/htroot/Banner.javasimilarity index 99%rename from htroot/Banner.javarename to source/net/yacy/htroot/Banner.javaindex 4f8667edc..93c74eb70 100644--- a/htroot/Banner.java+++ b/source/net/yacy/htroot/Banner.java@@ -25,6 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.awt.image.BufferedImage;import java.io.File;import java.io.IOException;diff --git a/htroot/BlacklistCleaner_p.java b/source/net/yacy/htroot/BlacklistCleaner_p.javasimilarity index 98%rename from htroot/BlacklistCleaner_p.javarename to source/net/yacy/htroot/BlacklistCleaner_p.javaindex e0a23134e..7ea0f2b22 100644--- a/htroot/BlacklistCleaner_p.java+++ b/source/net/yacy/htroot/BlacklistCleaner_p.java@@ -29,6 +29,8 @@// javac -classpath .:../classes BlacklistCleaner_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import java.util.ArrayList;import java.util.Collection;@@ -48,9 +50,9 @@ import net.yacy.data.ListManager;import net.yacy.document.parser.html.CharacterCoding;import net.yacy.kelondro.util.FileUtils;import net.yacy.repository.Blacklist;-import net.yacy.repository.BlacklistHostAndPath;import net.yacy.repository.Blacklist.BlacklistError;import net.yacy.repository.Blacklist.BlacklistType;+import net.yacy.repository.BlacklistHostAndPath;import net.yacy.search.Switchboard;import net.yacy.search.query.SearchEventCache;import net.yacy.server.serverObjects;@@ -160,7 +162,7 @@ public class BlacklistCleaner_p {/*** Adds a list of blacklist to the server objects properties which are used* to display the blacklist in the HTML page belonging to this servlet.- *+ ** @param prop* Server objects properties object.* @param lists@@ -206,7 +208,7 @@ public class BlacklistCleaner_p {* {@link getByPrefix(de.anomic.server.serverObjects, java.lang.String,* boolean, boolean)} which has been created to make it easier to understand* the code.- *+ ** @param post* All POST values.* @param prefix@@ -225,7 +227,7 @@ public class BlacklistCleaner_p {* sent and returns them as an array. This method is only a wrapper for* {@link getByPrefix(de.anomic.server.serverObjects, java.lang.String,* boolean, boolean)}.- *+ ** @param post* All POST values.* @param prefix@@ -245,7 +247,7 @@ public class BlacklistCleaner_p {* boolean)} and {@link getValuesByPrefix(de.anomic.server.serverObjects,* java.lang.String prefix, boolean)} which have been crested to make it* easier to understand the code.- *+ ** @param post* @param prefix* @param useKeys@@ -281,7 +283,7 @@ public class BlacklistCleaner_p {/*** Finds illegal entries in black list.- *+ ** @param blacklistToUse* The blacklist to be checked.* @param allowRegex@@ -292,7 +294,7 @@ public class BlacklistCleaner_p {* error code as value.*/private static Map<String, BlacklistError> getIllegalEntries(- final String blacklistToUse,+ final String blacklistToUse,final boolean allowRegex) {final Map<String, BlacklistError> illegalEntries = new HashMap<>();final Set<String> legalEntries = new HashSet<>();@@ -325,7 +327,7 @@ public class BlacklistCleaner_p {/*** Removes existing entries from a blacklist.- *+ ** @param blacklistToUse* The blacklist which contains the* @param supportedBlacklistTypes@@ -371,7 +373,7 @@ public class BlacklistCleaner_p {/*** Changes existing entry in a blacklist.- *+ ** @param blacklistToUse* The blacklist which contains the entry.* @param supportedBlacklistTypes@@ -405,7 +407,7 @@ public class BlacklistCleaner_p {blacklistToUse)) {try {Switchboard.urlBlacklist.add(s, blacklistToUse, newEntries);- } catch (PunycodeException e) {+ } catch (final PunycodeException e) {ConcurrentLog.warn(APP_NAME,"Unable to add blacklist entry to blacklist "+ s, e);diff --git a/htroot/BlacklistImpExp_p.java b/source/net/yacy/htroot/BlacklistImpExp_p.javasimilarity index 94%rename from htroot/BlacklistImpExp_p.javarename to source/net/yacy/htroot/BlacklistImpExp_p.javaindex 0235cd75f..3cc763b41 100644--- a/htroot/BlacklistImpExp_p.java+++ b/source/net/yacy/htroot/BlacklistImpExp_p.java@@ -29,6 +29,8 @@// javac -classpath .:../classes Blacklist_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.Iterator;import java.util.List;import java.util.TreeMap;@@ -72,7 +74,7 @@ public class BlacklistImpExp_p {if (seed != null) hostList.put(seed.get(Seed.NAME, "nameless"),seed.hash);}- for (String peername : hostList.keySet()) {+ for (final String peername : hostList.keySet()) {final String Hash = hostList.get(peername);prop.putHTML(DISABLED + "otherHosts_" + peerCount + "_hash", Hash);prop.putXML(DISABLED + "otherHosts_" + peerCount + "_name", peername);@@ -89,7 +91,7 @@ public class BlacklistImpExp_p {prop.put("disabled", (blacklistToUse == null) ? "1" : "0");int count = 0;- for (String element : dirlist) {+ for (final String element : dirlist) {prop.putHTML("blackListNames_" + count + "_blackListName", element);count++;}diff --git a/htroot/BlacklistTest_p.java b/source/net/yacy/htroot/BlacklistTest_p.javasimilarity index 99%rename from htroot/BlacklistTest_p.javarename to source/net/yacy/htroot/BlacklistTest_p.javaindex 7eb96d272..5e823d2b7 100644--- a/htroot/BlacklistTest_p.java+++ b/source/net/yacy/htroot/BlacklistTest_p.java@@ -29,6 +29,8 @@// javac -classpath .:../classes Blacklist_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.net.MalformedURLException;import net.yacy.cora.document.id.DigestURL;diff --git a/htroot/Blacklist_p.java b/source/net/yacy/htroot/Blacklist_p.javasimilarity index 98%rename from htroot/Blacklist_p.javarename to source/net/yacy/htroot/Blacklist_p.javaindex 2218c63c1..47b9d5a40 100644--- a/htroot/Blacklist_p.java+++ b/source/net/yacy/htroot/Blacklist_p.java@@ -29,6 +29,8 @@// javac -classpath .:../classes Blacklist_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.util.Arrays;@@ -54,13 +56,13 @@ import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;public class Blacklist_p {-+private final static String EDIT = "edit_";private final static String DISABLED = "disabled_";private final static String BLACKLIST = "blackLists_";private final static String BLACKLIST_MOVE = "blackListsMove_";private final static String BLACKLIST_SHARED = "BlackLists.Shared";-+public static serverObjects respond(final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {// load all blacklist files located in the directory@@ -173,7 +175,7 @@ public class Blacklist_p {ListManager.removeFromListSet(supportedBlacklistType + ".BlackLists",blacklistToUse);}}-+Switchboard.urlBlacklist.clear();ListManager.reloadBlacklists();@@ -204,7 +206,7 @@ public class Blacklist_p {blacklistToUse = post.get("currentBlacklist", "").trim();final String[] selectedBlacklistEntries = post.getAll("selectedEntry.*");-+// store this call as api callListManager.switchboard.tables.recordAPICall(post,"Blacklist_p.html",@@ -258,7 +260,7 @@ public class Blacklist_p {final String targetBlacklist = post.get("targetBlacklist");final String[] selectedBlacklistEntries = post.getAll("selectedEntry.*");-+// store this call as api callListManager.switchboard.tables.recordAPICall(post,@@ -280,7 +282,7 @@ public class Blacklist_p {prop.put(serverObjects.ACTION_LOCATION, temp);return prop;}-+if (!BlacklistHelper.addBlacklistEntry(targetBlacklist, selectedBlacklistEntry)) {prop.put(DISABLED + EDIT + "moveError", true);break;@@ -316,9 +318,9 @@ public class Blacklist_p {for (final Entry<String, String> selectedEntry : selectedBlacklistEntries.entrySet()) {final String editedEntryValue = editedBlacklistEntries.get(selectedEntry.getKey().replace("selectedBlacklistEntry.", "editedBlacklistEntry."));-+final String preparedNewEntry = prepareNormalizedEntry(editedEntryValue);-+if (!normalizeEntry(selectedEntry.getValue()).equals(preparedNewEntry)) {/* Add first, to detect any eventual syntax errors before removing the old entry */@@ -338,15 +340,15 @@ public class Blacklist_p {} else {/* At least one error occurred : display again entries with errors for edition */prop.put(DISABLED + EDIT + "editError", true);-+final int maxDisplayedErrors = 10;-+int i = 0;for (final Entry<String, String> selected2Edited : selected2EditedErrors.entrySet()) {/* We do not use here putHTML as we don't want '+' characters to be interpreted as application/x-www-form-urlencoded encoding */prop.put(DISABLED + EDIT + "editList_" + i + "_item", CharacterCoding.unicode2html(selected2Edited.getKey(), true));prop.put(DISABLED + EDIT + "editList_" + i + "_count", i);-+/* We do not use here putHTML as we don't want '+' characters to be interpreted as application/x-www-form-urlencoded encoding */if(i < maxDisplayedErrors) {prop.put(DISABLED + EDIT + "editError_list_" + i + "_item", CharacterCoding.unicode2html(selected2Edited.getValue(), true));@@ -365,7 +367,7 @@ public class Blacklist_p {prop.put(DISABLED + EDIT + "editList", selected2EditedErrors.size());prop.put(DISABLED + EDIT + "editError_list", Math.min(maxDisplayedErrors, selected2EditedErrors.size()));}-+// else return entry to be edited} else {@@ -396,7 +398,7 @@ public class Blacklist_p {if (blacklistToUse != null) {int entryCount = 0;// we should use same load methode as during internal init (even if we only need a simple array to display content)- String[] sortedlist = BlacklistHelper.blacklistToSortedArray(blacklistToUse);+ final String[] sortedlist = BlacklistHelper.blacklistToSortedArray(blacklistToUse);// display themboolean dark = true;int offset = 0;@@ -423,13 +425,13 @@ public class Blacklist_p {if (nextEntry.charAt(0) == '#') {continue;}-+/** Decode the entry for easier reading of paths with non ascii characters */final int slashPos = nextEntry.indexOf('/', 0);if(slashPos > 0) {nextEntry = nextEntry.substring(0, slashPos + 1) + MultiProtocolURL.unescapePath(nextEntry.substring(slashPos + 1));}-+prop.put(DISABLED + EDIT + "Itemlist_" + entryCount + "_dark", dark ? "1" : "0");dark = !dark;/* We do not use here putHTML as we don't want '+' characters to be interpreted as application/x-www-form-urlencoded encoding */@@ -440,7 +442,7 @@ public class Blacklist_p {prop.put(DISABLED + EDIT + "Itemlist", entryCount);// create selection of sublist- int[] navbar = new int[7];+ final int[] navbar = new int[7];// generate array of start index for navbar ( -1 = early endmark)// [0] [-2*size] [-1*size] [offset] [+1*size] [+2*size] [end]if (size > 0 && sortedlist.length > 0) {@@ -476,7 +478,7 @@ public class Blacklist_p {prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_tvalue", end);if (navbar[entryCount] == offset) {prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_selected", 1);- }+ }entryCount++;}} else {@@ -578,7 +580,7 @@ public class Blacklist_p {* BlacklistHelper.addBlacklistEntry()*/}- String path = MultiProtocolURL.escapePathPattern(entry.substring(slashPos + 1));+ final String path = MultiProtocolURL.escapePathPattern(entry.substring(slashPos + 1));return host + "/" + path;}diff --git a/htroot/Blog.java b/source/net/yacy/htroot/Blog.javasimilarity index 99%rename from htroot/Blog.javarename to source/net/yacy/htroot/Blog.javaindex 5f88bbb47..01d3f49f4 100644--- a/htroot/Blog.java+++ b/source/net/yacy/htroot/Blog.java@@ -30,6 +30,8 @@// javac -classpath .:../classes Blog.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.text.DateFormat;import java.util.Date;import java.util.HashMap;diff --git a/htroot/BlogComments.java b/source/net/yacy/htroot/BlogComments.javasimilarity index 98%rename from htroot/BlogComments.javarename to source/net/yacy/htroot/BlogComments.javaindex d9137c2e9..8dbb43536 100644--- a/htroot/BlogComments.java+++ b/source/net/yacy/htroot/BlogComments.java@@ -29,12 +29,16 @@// javac -classpath .:../Classes Blacklist_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.io.PrintWriter;import java.util.Date;import java.util.Iterator;+import com.google.common.io.Files;+import net.yacy.cora.document.encoding.UTF8;import net.yacy.cora.protocol.Domains;import net.yacy.cora.protocol.HeaderFramework;@@ -42,10 +46,10 @@ import net.yacy.cora.protocol.RequestHeader;import net.yacy.cora.util.CommonPattern;import net.yacy.cora.util.ConcurrentLog;import net.yacy.data.BlogBoard;+import net.yacy.data.BlogBoard.BlogEntry;import net.yacy.data.BlogBoardComments;import net.yacy.data.MessageBoard;import net.yacy.data.UserDB;-import net.yacy.data.BlogBoard.BlogEntry;import net.yacy.peers.Network;import net.yacy.peers.Seed;import net.yacy.search.Switchboard;@@ -53,8 +57,6 @@ import net.yacy.search.SwitchboardConstants;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;-import com.google.common.io.Files;-public class BlogComments {@@ -86,7 +88,7 @@ public class BlogComments {}}- String pagename = post.get("page", DEFAULT_PAGE);+ final String pagename = post.get("page", DEFAULT_PAGE);final String ip = post.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);String strAuthor = post.get("author", "anonymous");@@ -134,7 +136,7 @@ public class BlogComments {sb.blogCommentDB.write(sb.blogCommentDB.newEntry(commentID, subject, author, ip, date, content));prop.putHTML(serverObjects.ACTION_LOCATION,"BlogComments.html?page=" + pagename);- MessageBoard.entry msgEntry = sb.messageDB.newEntry(+ final MessageBoard.entry msgEntry = sb.messageDB.newEntry("blogComment",strAuthor,sb.peers.mySeed().hash,diff --git a/htroot/Bookmarks.java b/source/net/yacy/htroot/Bookmarks.javasimilarity index 97%rename from htroot/Bookmarks.javarename to source/net/yacy/htroot/Bookmarks.javaindex 4c9b78d80..9ac240260 100644--- a/htroot/Bookmarks.java+++ b/source/net/yacy/htroot/Bookmarks.java@@ -29,6 +29,8 @@// javac -classpath .:../Classes Blacklist_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.net.MalformedURLException;@@ -48,10 +50,10 @@ import net.yacy.cora.protocol.RequestHeader;import net.yacy.cora.util.ConcurrentLog;import net.yacy.data.BookmarkHelper;import net.yacy.data.BookmarksDB;-import net.yacy.data.ListManager;-import net.yacy.data.UserDB;import net.yacy.data.BookmarksDB.Bookmark;import net.yacy.data.BookmarksDB.Tag;+import net.yacy.data.ListManager;+import net.yacy.data.UserDB;import net.yacy.document.Document;import net.yacy.document.Parser;import net.yacy.http.servlets.YaCyDefaultServlet;@@ -180,7 +182,7 @@ public class Bookmarks {}bookmark.setTags(tags, true);-+if (post.containsKey("query")) {bookmark.setProperty(Bookmark.BOOKMARK_QUERY, post.get("query"));bookmark.setTimeStamp(System.currentTimeMillis());@@ -193,7 +195,7 @@ public class Bookmarks {prop.put("mode", "2");prop.put("display", "1");display = 1;- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));if (urlHash.isEmpty()) {prop.put("mode_edit", "0"); // create modeprop.putHTML("mode_title", post.get("title"));@@ -204,7 +206,7 @@ public class Bookmarks {prop.put("mode_public", "0");prop.put("mode_feed", "0");} else {- BookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(urlHash);+ final BookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(urlHash);if (bookmark == null) {// try to get the bookmark from the LURL database@@ -325,7 +327,7 @@ public class Bookmarks {count = 0;while(count < max_count && it.hasNext()) {- Bookmark bookmark = sb.bookmarksDB.getBookmark(it.next());+ final Bookmark bookmark = sb.bookmarksDB.getBookmark(it.next());if (bookmark != null){if (bookmark.getFeed() && isAdmin) {@@ -390,7 +392,7 @@ public class Bookmarks {count = recurseFolders(BookmarkHelper.getFolderList("/", sb.bookmarksDB.getTagIterator(isAdmin)), "/", 0, true, "");prop.put("display_folderlist", count);- BusyThread bt = sb.getThread("autosearch");+ final BusyThread bt = sb.getThread("autosearch");if (bt != null) {prop.put("display_autosearchrunning","1");prop.put("display_autosearchrunning_msg", "" );@@ -399,7 +401,7 @@ public class Bookmarks {prop.put("display_autosearchrunning_msg", "autosearch will terminate");prop.put("display_autosearchrunning","0");}- int jobs = bt.getJobCount();+ final int jobs = bt.getJobCount();prop.put("display_autosearchrunning_jobcount", jobs);int cnt=0;String qstr = "";@@ -487,7 +489,7 @@ public class Bookmarks {final Iterator<String> bit = sb.bookmarksDB.getBookmarksIterator(fn, isAdmin);while (bit.hasNext()) {- Bookmark bookmark = sb.bookmarksDB.getBookmark(bit.next());+ final Bookmark bookmark = sb.bookmarksDB.getBookmark(bit.next());if (bookmark == null) break;prop.put("display_folderlist_" + count + "_folder", "<li><a href=\"" + bookmark.getUrl() + "\" title=\"" + bookmark.getDescription() + "\">" + bookmark.getTitle() + "</a></li>");count++;diff --git a/htroot/CacheResource_p.java b/source/net/yacy/htroot/CacheResource_p.javasimilarity index 87%rename from htroot/CacheResource_p.javarename to source/net/yacy/htroot/CacheResource_p.javaindex d4b189677..eae69d8ea 100644--- a/htroot/CacheResource_p.java+++ b/source/net/yacy/htroot/CacheResource_p.java@@ -22,6 +22,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.IOException;import java.net.MalformedURLException;import java.nio.ByteBuffer;@@ -47,13 +49,13 @@ import net.yacy.server.servletProperties;public class CacheResource_p {public static Object respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {- Switchboard sb = (Switchboard) env;+ final Switchboard sb = (Switchboard) env;final servletProperties prop = new servletProperties();prop.put("resource", new byte[0]);if (post == null) return prop;- boolean load = post.getBoolean("load");+ final boolean load = post.getBoolean("load");final String u = post.get("url", "");DigestURL url;try {@@ -71,7 +73,7 @@ public class CacheResource_p {final Response response = sb.loader.load(sb.loader.request(url, false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent);responseHeader = response.getResponseHeader();resource = response.getContent();- } catch (IOException e) {+ } catch (final IOException e) {return prop;}} else return prop;@@ -98,15 +100,15 @@ public class CacheResource_p {} else {decoderCharset = StandardCharsets.UTF_8;}- CharBuffer cb = decoderCharset.decode(ByteBuffer.wrap(resource));- ByteBuffer x = StandardCharsets.UTF_16.encode(cb); // encode to a default java string (which uses utf-16 and is handled correct for servlet content)+ final CharBuffer cb = decoderCharset.decode(ByteBuffer.wrap(resource));+ final ByteBuffer x = StandardCharsets.UTF_16.encode(cb); // encode to a default java string (which uses utf-16 and is handled correct for servlet content)prop.put("resource", x.asCharBuffer().toString());} else {prop.put("resource", resource);}- // add resource+ // add resourcereturn prop;}}diff --git a/htroot/Collage.java b/source/net/yacy/htroot/Collage.javasimilarity index 99%rename from htroot/Collage.javarename to source/net/yacy/htroot/Collage.javaindex 95233098c..f5aa8c0ed 100644--- a/htroot/Collage.java+++ b/source/net/yacy/htroot/Collage.java@@ -22,6 +22,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Random;import net.yacy.cora.document.id.MultiProtocolURL;diff --git a/htroot/ConfigAccountList_p.java b/source/net/yacy/htroot/ConfigAccountList_p.javasimilarity index 91%rename from htroot/ConfigAccountList_p.javarename to source/net/yacy/htroot/ConfigAccountList_p.javaindex ea772af4b..7a3dff960 100644--- a/htroot/ConfigAccountList_p.java+++ b/source/net/yacy/htroot/ConfigAccountList_p.java@@ -18,6 +18,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Iterator;import net.yacy.cora.date.GenericFormatter;@@ -75,8 +77,8 @@ public class ConfigAccountList_p {long percent;if (entry.getTrafficLimit() != null) {- long limit = entry.getTrafficLimit();- long used = entry.getTrafficSize();+ final long limit = entry.getTrafficLimit();+ final long used = entry.getTrafficSize();percent = used * 100 / limit;prop.put("userlist_" + numUsers + "_time", percent);} else {@@ -85,8 +87,8 @@ public class ConfigAccountList_p {percent = -1;if (entry.getTimeLimit() > 0) {- long limit = entry.getTimeLimit();- long used = entry.getTimeUsed();+ final long limit = entry.getTimeLimit();+ final long used = entry.getTimeUsed();percent = used * 100 / limit;prop.put("userlist_" + numUsers + "_traffic", percent);} else {diff --git a/htroot/ConfigAccounts_p.java b/source/net/yacy/htroot/ConfigAccounts_p.javasimilarity index 95%rename from htroot/ConfigAccounts_p.javarename to source/net/yacy/htroot/ConfigAccounts_p.javaindex cd5562999..053ed4c68 100644--- a/htroot/ConfigAccounts_p.java+++ b/source/net/yacy/htroot/ConfigAccounts_p.java@@ -29,6 +29,8 @@//javac -classpath .:../Classes Message.java//if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.EnumMap;import java.util.HashMap;import java.util.Iterator;@@ -46,14 +48,14 @@ import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;public class ConfigAccounts_p {-+public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {final serverObjects prop = new serverObjects();-+/* Acquire a transaction token for the next POST form submission */prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));-+final Switchboard sb = (Switchboard) env;UserDB.Entry entry = null;@@ -64,7 +66,7 @@ public class ConfigAccounts_p {TransactionManager.checkPostTransaction(header, post);sb.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_All_PAGES, post.getBoolean(SwitchboardConstants.ADMIN_ACCOUNT_All_PAGES));}-+if (post != null && post.containsKey("setAdmin")) {TransactionManager.checkPostTransaction(header, post);localhostAccess = post.get("access", "").equals("localhost");@@ -74,13 +76,13 @@ public class ConfigAccounts_p {int inputerror=0;// may be overwritten if new password is givenif (user.length() > 0 && pw1.equals(pw2)) {- String oldusername = env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,user);+ final String oldusername = env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,user);// check passed. set account:// old: // env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, Digest.encodeMD5Hex(Base64Order.standardCoder.encodeString(user + ":" + pw1)));env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, sb.encodeDigestAuth(user, pw1));env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,user);// make sure server accepts new credentials- YaCyHttpServer jhttpserver = (YaCyHttpServer)sb.getHttpServer();+ final YaCyHttpServer jhttpserver = (YaCyHttpServer)sb.getHttpServer();if (!user.equals(oldusername)) jhttpserver.removeUser(oldusername);jhttpserver.resetUser(user);} else {@@ -91,7 +93,7 @@ public class ConfigAccounts_p {inputerror = 2; // password match error}prop.put("error", inputerror);- }+ }}if (inputerror == 0) {@@ -104,7 +106,7 @@ public class ConfigAccounts_p {env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "");}}- }+ }}if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").isEmpty() && !env.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false)) {@@ -259,7 +261,7 @@ public class ConfigAccounts_p {prop.putHTML("text_username", username);prop.put("text", "2");}//edit user-+prop.putHTML("username", username);if (entry != null) {//TODO: set username read-only in htmldiff --git a/htroot/ConfigAppearance_p.java b/source/net/yacy/htroot/ConfigAppearance_p.javasimilarity index 93%rename from htroot/ConfigAppearance_p.javarename to source/net/yacy/htroot/ConfigAppearance_p.javaindex ea5bae159..be32583fb 100644--- a/htroot/ConfigAppearance_p.java+++ b/source/net/yacy/htroot/ConfigAppearance_p.java@@ -29,6 +29,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.BufferedWriter;import java.io.File;import java.io.FileWriter;@@ -39,6 +41,8 @@ import java.util.Iterator;import java.util.List;import java.util.Map;+import com.google.common.io.Files;+import net.yacy.cora.document.id.DigestURL;import net.yacy.cora.protocol.ClientIdentification;import net.yacy.cora.protocol.RequestHeader;@@ -48,8 +52,6 @@ import net.yacy.search.SwitchboardConstants;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;-import com.google.common.io.Files;-public class ConfigAppearance_p {@@ -70,7 +72,7 @@ public class ConfigAppearance_p {}if (post != null) {- String selectedSkin = post.get("skin");+ final String selectedSkin = post.get("skin");if (post.containsKey("use_button") && selectedSkin != null) {/* Only change skin if filename is contained in list of filesnames@@ -138,7 +140,7 @@ public class ConfigAppearance_p {}}- String currentSkin = env.getConfig("currentSkin", "default");+ final String currentSkin = env.getConfig("currentSkin", "default");prop.putHTML("currentskin", currentSkin);// reread skinsskinFiles = FileUtils.getDirListing(skinPath, SKIN_FILENAME_FILTER);@@ -146,7 +148,7 @@ public class ConfigAppearance_p {int count = 0;for (final String skinFile : skinFiles) {if (skinFile.endsWith(".css")) {- String skinName = skinFile.substring(0, skinFile.length() - 4);+ final String skinName = skinFile.substring(0, skinFile.length() - 4);prop.put("skinlist_" + count + "_file", skinFile);prop.put("skinlist_" + count + "_name", skinName);prop.put("skinlist_" + count + "_selected", skinName.equals(currentSkin) ? 1 : 0);@@ -156,7 +158,7 @@ public class ConfigAppearance_p {prop.put("skinlist", count);// write colors from generic skin- Iterator<String> i = env.configKeys();+ final Iterator<String> i = env.configKeys();while (i.hasNext()) {final String key = i.next();if (key.startsWith("color_")) prop.put(key, env.getConfig(key, "#000000").substring(1));diff --git a/htroot/ConfigBasic.java b/source/net/yacy/htroot/ConfigBasic.javasimilarity index 97%rename from htroot/ConfigBasic.javarename to source/net/yacy/htroot/ConfigBasic.javaindex 16520dc0b..40b0a2b58 100644--- a/htroot/ConfigBasic.java+++ b/source/net/yacy/htroot/ConfigBasic.java@@ -28,6 +28,8 @@// javac -classpath .:../classes ConfigBasic_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import java.io.FileNotFoundException;import java.io.IOException;@@ -58,7 +60,7 @@ public class ConfigBasic {private static final int NEXTSTEP_PEERNAME = 2;private static final int NEXTSTEP_PEERPORT = 3;private static final int NEXTSTEP_RECONNECT = 4;-+public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws FileNotFoundException, IOException {// return variable that accumulates replacements@@ -66,17 +68,17 @@ public class ConfigBasic {final serverObjects prop = new serverObjects();final File langPath = new File(sb.getAppPath("locale.source", "locales").getAbsolutePath());String lang = env.getConfig("locale.language", "browser");-+final int authentication = sb.adminAuthenticated(header);if (authentication < 2) {// must authenticateprop.authenticationRequired();return prop;}-+/* For authenticated users only : acquire a transaction token for the next POST form submission */prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));-+if ((sb.peers.mySeed().isVirgin()) || (sb.peers.mySeed().isJunior())) {new OnePeerPingBusyThread(sb.yc).start();}@@ -93,23 +95,23 @@ public class ConfigBasic {if(post.containsKey("set")) {sb.tables.recordAPICall(post, "ConfigBasic.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "basic settings");}-+// language settingsif(post.containsKey("language") && !lang.equals(post.get("language", "default"))) {if(new TranslatorXliff().changeLang(env, langPath, post.get("language", "default") + ".lng")) {prop.put("changedLanguage", "1");}}-+// port settingsif(post.getInt("port", 0) > 1023) {port = post.getLong("port", 8090);ssl = post.getBoolean("withssl");}-+// peer name settingspeerName = post.get("peername", "");-+// UPnP configif(post.containsKey("port")) { // hack to allow checkboxupnp = post.containsKey("enableUpnp");@@ -122,11 +124,11 @@ public class ConfigBasic {}}}-+if (ssl) {prop.put("withsslenabled_sslport", env.getHttpServer().getSslPort());}-+if (peerName != null && peerName.length() > 0) {peerName = peerName.replace(' ', '-');}@@ -147,7 +149,7 @@ public class ConfigBasic {final YaCyHttpServer theServerCore = env.getHttpServer();env.setConfig(SwitchboardConstants.SERVER_PORT, port);env.setConfig("server.https", ssl);-+// redirect the browser to the new portreconnect = true;@@ -166,7 +168,7 @@ public class ConfigBasic {prop.put("nextStep_host", host);prop.put("reconnect_port", port);prop.put("nextStep_port", port);- prop.put("reconnect_sslSupport", theServerCore.withSSL() ? "1" : "0");+ prop.put("reconnect_sslSupport", theServerCore.withSSL() ? "1" : "0");prop.put("nextStep_sslSupport", theServerCore.withSSL() ? "1" : "0");// generate new shortcut (used for Windows)@@ -187,8 +189,8 @@ public class ConfigBasic {if (post != null && post.containsKey("usecase")) {/* Settings will be modified : check this is a valid transaction using HTTP POST method */TransactionManager.checkPostTransaction(header, post);-- boolean hasNonEmptyRemoteSolr = sb.index.fulltext().connectedRemoteSolr()++ final boolean hasNonEmptyRemoteSolr = sb.index.fulltext().connectedRemoteSolr()&& (sb.index.fulltext().collectionSize() > 0 || sb.index.fulltext().webgraphSize() > 0);if ("freeworld".equals(post.get("usecase", "")) && !"freeworld".equals(networkName)) {if("intranet".equals(networkName) && hasNonEmptyRemoteSolr) {@@ -198,7 +200,7 @@ public class ConfigBasic {if(hasNonEmptyRemoteSolr) {/* One or more non empty remote Solr(s) attached : warn the user even if not coming from intranet as indexed documents may be irrelevant for the new mode */prop.put("setUseCase_switchWarning", 2);- }+ }// switch to freeworld networksb.setConfig(SwitchboardConstants.CORE_SERVICE_RWI, true);sb.switchNetwork("defaults/yacy.network.freeworld.unit");@@ -219,8 +221,8 @@ public class ConfigBasic {if(hasNonEmptyRemoteSolr) {/* One or more non empty remote Solr(s) attached : warn the user even if not coming from intranet as indexed documents may be irrelevant for the new mode */prop.put("setUseCase_switchWarning", 2);- }-+ }+// switch to webportal networksb.setConfig(SwitchboardConstants.CORE_SERVICE_RWI, false);sb.switchNetwork("defaults/yacy.network.webportal.unit");@@ -237,7 +239,7 @@ public class ConfigBasic {if(hasNonEmptyRemoteSolr) {/* One or more non empty remote Solr(s) attached : warn the user as the indexed documents may be public external resources out of scope for intranet/localhost domain. */prop.put("setUseCase_switchWarning", 1);- }+ }// switch to intranet networksb.setConfig(SwitchboardConstants.CORE_SERVICE_RWI, false);sb.switchNetwork("defaults/yacy.network.intranet.unit");@@ -324,7 +326,7 @@ public class ConfigBasic {}// set label class (green background) for active translationif (lang.equals("browser")) {- List<String> l = Translator.activeTranslations();+ final List<String> l = Translator.activeTranslations();prop.put("active_zh", l.contains("zh") ? "2" : "1");prop.put("active_de", l.contains("de") ? "2" : "1");prop.put("active_fr", l.contains("fr") ? "2" : "1");@@ -336,7 +338,7 @@ public class ConfigBasic {prop.put("active_uk", l.contains("uk") ? "2" : "1");prop.put("active_es", l.contains("es") ? "2" : "1");prop.put("active_en", "2");-+} else {prop.put("active_de", "0");prop.put("active_fr", "0");diff --git a/htroot/ConfigHTCache_p.java b/source/net/yacy/htroot/ConfigHTCache_p.javasimilarity index 98%rename from htroot/ConfigHTCache_p.javarename to source/net/yacy/htroot/ConfigHTCache_p.javaindex 827dab17a..9608074ff 100644--- a/htroot/ConfigHTCache_p.java+++ b/source/net/yacy/htroot/ConfigHTCache_p.java@@ -26,6 +26,8 @@// javac -classpath .:../classes ProxyIndexingMonitor_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.text.NumberFormat;@@ -63,14 +65,14 @@ public class ConfigHTCache_p {final int newProxyCacheSize = Math.max(post.getInt("maxCacheSize", 64), 0);env.setConfig(SwitchboardConstants.PROXY_CACHE_SIZE, newProxyCacheSize);Cache.setMaxCacheSize(newProxyCacheSize * 1024L * 1024L);-+/* Compression level*//* Ensure a value within the range supported by the Deflater class */final int newCompressionLevel = Math.max(Deflater.NO_COMPRESSION, Math.min(Deflater.BEST_COMPRESSION,post.getInt("compressionLevel", SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT)));env.setConfig(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL, newCompressionLevel);Cache.setCompressionLevel(newCompressionLevel);-+/* Synchronization lock timeout */final long newLockTimeout = Math.max(10, Math.min(60000,post.getLong("lockTimeout", SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT)));@@ -88,21 +90,21 @@ public class ConfigHTCache_p {}prop.put("HTCachePath", env.getConfig(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT));-+/* Compression levels */final int configuredCompressionLevel = env.getConfigInt(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL,SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT);int levelsCount = 0;for(int level = Deflater.NO_COMPRESSION; level <= Deflater.BEST_COMPRESSION; level++) {if(level == configuredCompressionLevel) {- prop.put("compressionLevels_" + levelsCount + "_selected", "1");+ prop.put("compressionLevels_" + levelsCount + "_selected", "1");} else {prop.put("compressionLevels_" + levelsCount + "_selected", "0");}prop.put("compressionLevels_" + levelsCount + "_value", level);prop.put("compressionLevels_" + levelsCount + "_name", level);if(level == Deflater.NO_COMPRESSION) {- prop.put("compressionLevels_" + levelsCount + "_name", "0 - No compression");+ prop.put("compressionLevels_" + levelsCount + "_name", "0 - No compression");} else if(level == Deflater.BEST_SPEED) {prop.put("compressionLevels_" + levelsCount + "_name", Deflater.BEST_SPEED + " - Best speed");} else if(level == Deflater.BEST_COMPRESSION) {@@ -111,7 +113,7 @@ public class ConfigHTCache_p {levelsCount++;}prop.put("compressionLevels", levelsCount);-+prop.put("lockTimeout", env.getConfigLong(SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT,SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT));prop.put("actualCacheSize", Cache.getActualCacheSize() / 1024 / 1024);@@ -124,7 +126,7 @@ public class ConfigHTCache_p {prop.put("hits", hits);prop.put("requests", totalRequests);prop.put("hitRate", NumberFormat.getPercentInstance().format(Cache.getHitRate()));-+// return rewrite propertiesreturn prop;}diff --git a/htroot/ConfigHeuristics_p.java b/source/net/yacy/htroot/ConfigHeuristics_p.javasimilarity index 95%rename from htroot/ConfigHeuristics_p.javarename to source/net/yacy/htroot/ConfigHeuristics_p.javaindex f9c549615..90fb7536b 100644--- a/htroot/ConfigHeuristics_p.java+++ b/source/net/yacy/htroot/ConfigHeuristics_p.java@@ -26,16 +26,9 @@// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA-import java.io.File;--import net.yacy.cora.protocol.ClientIdentification;-import net.yacy.cora.protocol.RequestHeader;-import net.yacy.cora.storage.Configuration;-import net.yacy.cora.util.ConcurrentLog;-import net.yacy.crawler.robots.RobotsTxtEntry;-import net.yacy.data.WorkTables;-import net.yacy.search.Switchboard;+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.net.MalformedURLException;import java.util.Iterator;@@ -43,9 +36,15 @@ import java.util.Locale;import net.yacy.cora.document.id.MultiProtocolURL;import net.yacy.cora.federate.FederateSearchManager;-import net.yacy.cora.federate.solr.SchemaConfiguration;+import net.yacy.cora.protocol.ClientIdentification;+import net.yacy.cora.protocol.RequestHeader;+import net.yacy.cora.storage.Configuration;import net.yacy.cora.storage.Files;+import net.yacy.cora.util.ConcurrentLog;+import net.yacy.crawler.robots.RobotsTxtEntry;+import net.yacy.data.WorkTables;+import net.yacy.search.Switchboard;import net.yacy.search.SwitchboardConstants;import net.yacy.search.schema.WebgraphSchema;import net.yacy.server.serverObjects;@@ -89,7 +88,7 @@ public class ConfigHeuristics_p {} else {osderrmsg = "Error: webgraph Solr index not enabled";}- } else {+ } else {osderrmsg = "Solr webgraph index needs to be available and fields target_rel_s, target_protocol_s, target_urlstub_s on";}}@@ -106,7 +105,7 @@ public class ConfigHeuristics_p {/* Check eventual robots.txt policy */RobotsTxtEntry robotsEntry = null;try {- MultiProtocolURL templateURL = new MultiProtocolURL(tmpurl);+ final MultiProtocolURL templateURL = new MultiProtocolURL(tmpurl);if (sb.robots != null) {robotsEntry = sb.robots.getEntry(templateURL,@@ -134,10 +133,10 @@ public class ConfigHeuristics_p {writeopensearchcfg (sb,post);}- if (post.containsKey("switchsolrfieldson")) {- final boolean metafieldavailable = sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.target_rel_s.name())+ if (post.containsKey("switchsolrfieldson")) {+ final boolean metafieldavailable = sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.target_rel_s.name())&& ( sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.target_protocol_s.name()) && sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.target_urlstub_s.name()) ) ;-+if (!metafieldavailable) {SchemaConfiguration.Entry entry;entry = sb.index.fulltext().getWebgraphConfiguration().get(WebgraphSchema.target_rel_s.name());@@ -156,7 +155,7 @@ public class ConfigHeuristics_p {sb.index.fulltext().getWebgraphConfiguration().commit();} catch (final IOException e) {}- sb.setConfig(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, true);+ sb.setConfig(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, true);}}@@ -168,7 +167,7 @@ public class ConfigHeuristics_p {if ((post.containsKey("resettodefaultosdlist") || !osdConfig.exists()) && osdDefaultConfig.exists()) {try {Files.copy(osdDefaultConfig, osdConfig);- File defdir = new File(sb.dataPath, "DATA/SETTINGS/federatecfg");+ final File defdir = new File(sb.dataPath, "DATA/SETTINGS/federatecfg");if (!defdir.exists()) {Files.copy(new File(sb.appPath, "defaults/federatecfg"), defdir);}@@ -192,12 +191,12 @@ public class ConfigHeuristics_p {final File f = new File (sb.getDataPath(),"DATA/SETTINGS/heuristicopensearch.conf");try {- Configuration p = new Configuration(f);+ final Configuration p = new Configuration(f);int c = 0;boolean dark = false;- Iterator<Configuration.Entry> i = p.entryIterator();+ final Iterator<Configuration.Entry> i = p.entryIterator();while (i.hasNext()) {- SchemaConfiguration.Entry e = i.next();+ final SchemaConfiguration.Entry e = i.next();prop.put("osdcfg_" + c + "_dark", dark ? 1 : 0);dark = !dark;prop.put("osdcfg_" + c + "_checked", e.enabled() ? 1 : 0);@@ -230,7 +229,7 @@ public class ConfigHeuristics_p {final File f = new File(sb.getDataPath(), "DATA/SETTINGS/heuristicopensearch.conf");try {- Configuration cfg = new Configuration(f);+ final Configuration cfg = new Configuration(f);final Iterator<Configuration.Entry> cfgentries = cfg.entryIterator();Configuration.Entry entry;boolean modified = false; // flag to remember changes@@ -267,7 +266,7 @@ public class ConfigHeuristics_p {} catch (final IOException e) {ConcurrentLog.warn("OpenSearchConnector", "file not found " + f.getAbsolutePath());}-+// re-read config (and create/update work table)if (sb.getConfigBool(SwitchboardConstants.HEURISTIC_OPENSEARCH, true)) {FederateSearchManager.getManager().init(f.getAbsolutePath());diff --git a/htroot/ConfigLanguage_p.java b/source/net/yacy/htroot/ConfigLanguage_p.javasimilarity index 95%rename from htroot/ConfigLanguage_p.javarename to source/net/yacy/htroot/ConfigLanguage_p.javaindex 8851e9121..0aea2f984 100644--- a/htroot/ConfigLanguage_p.java+++ b/source/net/yacy/htroot/ConfigLanguage_p.java@@ -29,7 +29,8 @@//javac -classpath .:../Classes Blacklist_p.java//if the shell's current path is HTROOT-import com.google.common.io.Files;+package net.yacy.htroot;+import java.io.File;import java.io.FileOutputStream;import java.io.IOException;@@ -40,6 +41,8 @@ import java.util.Iterator;import java.util.List;import java.util.Map;+import com.google.common.io.Files;+import net.yacy.cora.document.id.DigestURL;import net.yacy.cora.protocol.ClientIdentification;import net.yacy.cora.protocol.RequestHeader;@@ -57,7 +60,7 @@ public class ConfigLanguage_p {public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {final serverObjects prop = new serverObjects();- Switchboard sb = (Switchboard) env;+ final Switchboard sb = (Switchboard) env;final File langPath = new File(sb.getAppPath("locale.source", "locales").getAbsolutePath());//Fallback@@ -105,7 +108,7 @@ public class ConfigLanguage_p {try {final DigestURL u = new DigestURL(url);it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, null, null));- TranslatorXliff tx = new TranslatorXliff();+ final TranslatorXliff tx = new TranslatorXliff();File langFile = tx.getScratchFile(new File(langPath, u.getFileName()));try {try (@@ -116,16 +119,16 @@ public class ConfigLanguage_p {bw.write(it.next() + "\n");}}--++// convert downloaded xliff to internal lng filefinal String ext = Files.getFileExtension(langFile.getName());if (ext.equalsIgnoreCase("xlf") || ext.equalsIgnoreCase("xliff")) {- Map<String,Map<String,String>> lng = tx.loadTranslationsListsFromXliff(langFile);+ final Map<String,Map<String,String>> lng = tx.loadTranslationsListsFromXliff(langFile);langFile = new File(langPath, Files.getNameWithoutExtension(langFile.getName())+".lng");tx.saveAsLngFile(null, langFile, lng);}-+if (post.containsKey("use_lang") && "on".equals(post.get("use_lang"))) {tx.changeLang(env, langPath, langFile.getName());}@@ -153,7 +156,7 @@ public class ConfigLanguage_p {prop.put("langlist_1_name", ((langNames.get("default") == null) ? "default" : langNames.get("default")));prop.put("langlist_1_selected", sellang.equals("default") ? "selected=\"selected\"":" ");int count = 2; //+2 because of the virtual entry "browser" and "default" at top- for (final String langFile : langFiles) {+ for (final String langFile : langFiles) {final String langKey = langFile.substring(0, langFile.length() -4);final String langName = langNames.get(langKey);prop.put("langlist_" + (count) + "_file", langFile);diff --git a/htroot/ConfigNetwork_p.java b/source/net/yacy/htroot/ConfigNetwork_p.javasimilarity index 98%rename from htroot/ConfigNetwork_p.javarename to source/net/yacy/htroot/ConfigNetwork_p.javaindex 1f368149d..2ff40ec92 100644--- a/htroot/ConfigNetwork_p.java+++ b/source/net/yacy/htroot/ConfigNetwork_p.java@@ -25,6 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.File;import java.io.FileNotFoundException;import java.io.IOException;@@ -52,10 +54,10 @@ public class ConfigNetwork_pfinal Switchboard sb = (Switchboard) env;final serverObjects prop = new serverObjects();-+/* Acquire a transaction token for the next POST form submission */prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));-+int commit = 0;// load all options for network definitions@@ -75,7 +77,7 @@ public class ConfigNetwork_pif ( post.containsKey("changeNetwork") ) {/* Settings will be modified : check this is a valid transaction using HTTP POST method */TransactionManager.checkPostTransaction(header, post);-+String networkDefinition = post.get("networkDefinition", "defaults/yacy.network.freeworld.unit");final String networkDefinitionURL = post.get("networkDefinitionURL", "");if ( !networkDefinitionURL.equals("")) {@@ -91,7 +93,7 @@ public class ConfigNetwork_psb.switchNetwork(networkDefinition);}}-+if (post.containsKey("setEncryption")) {/** Settings will be modified : check this is a valid transaction using HTTP POST@@ -119,7 +121,7 @@ public class ConfigNetwork_p// DHT controlboolean indexDistribute = "on".equals(post.get("indexDistribute", ""));boolean indexReceive = "on".equals(post.get("indexReceive", ""));- boolean indexReceiveSearch = "on".equals(post.get("indexReceiveSearch", ""));+ final boolean indexReceiveSearch = "on".equals(post.get("indexReceiveSearch", ""));if ( !indexReceive ) {// remove heuristicssb.setConfig(SwitchboardConstants.HEURISTIC_SITE, false);@@ -194,7 +196,7 @@ public class ConfigNetwork_pprop.put("indexDistributeChecked", indexDistribute);prop.put("indexReceiveChecked", indexReceive);prop.put("indexReceiveSearchChecked", indexReceiveSearch);-+// set seed information directlysb.peers.mySeed().setFlagAcceptRemoteCrawl(sb.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false));sb.peers.mySeed().setFlagAcceptRemoteIndex(indexReceive);@@ -231,7 +233,7 @@ public class ConfigNetwork_pprop.put("networks_" + c++ + "_network", s);}prop.put("networks", c);-+// set encryption settingprop.put(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,diff --git a/htroot/ConfigParser_p.java b/source/net/yacy/htroot/ConfigParser_p.javasimilarity index 99%rename from htroot/ConfigParser_p.javarename to source/net/yacy/htroot/ConfigParser_p.javaindex 1294cfba4..e466d783b 100644--- a/htroot/ConfigParser_p.java+++ b/source/net/yacy/htroot/ConfigParser_p.java@@ -25,6 +25,8 @@// javac -classpath .:../Classes Settings_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.document.Parser;import net.yacy.document.TextParser;diff --git a/htroot/ConfigPortal.java b/source/net/yacy/htroot/ConfigPortal.javasimilarity index 98%rename from htroot/ConfigPortal.javarename to source/net/yacy/htroot/ConfigPortal.javaindex adeb9ab7c..a72abd910 100644--- a/htroot/ConfigPortal.java+++ b/source/net/yacy/htroot/ConfigPortal.java@@ -25,6 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;diff --git a/htroot/ConfigPortal_p.java b/source/net/yacy/htroot/ConfigPortal_p.javasimilarity index 97%rename from htroot/ConfigPortal_p.javarename to source/net/yacy/htroot/ConfigPortal_p.javaindex 8b1af4c2a..1cc28987f 100644--- a/htroot/ConfigPortal_p.java+++ b/source/net/yacy/htroot/ConfigPortal_p.java@@ -25,6 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.net.MalformedURLException;import java.time.Duration;import java.util.Properties;@@ -52,9 +54,9 @@ public class ConfigPortal_p {if (post != null) {/* Check this is a valid transaction */TransactionManager.checkPostTransaction(header, post);-+boolean cleanSearchCache = false;-+if (post.containsKey("popup")) {final String popup = post.get("popup", "status");if ("front".equals(popup)) {@@ -88,24 +90,24 @@ public class ConfigPortal_p {sb.setConfig("publicTopmenu", !post.containsKey("publicTopmenu") || post.getBoolean("publicTopmenu"));sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, !post.containsKey(SwitchboardConstants.PUBLIC_SEARCHPAGE) || post.getBoolean(SwitchboardConstants.PUBLIC_SEARCHPAGE));sb.setConfig("search.options", post.getBoolean("search.options"));-+final boolean oldJsResort = sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT);final boolean newJsResort = post.getBoolean(SwitchboardConstants.SEARCH_JS_RESORT);/* When this setting has changed we must clean up the search event cache as it affects how search results are retrieved */cleanSearchCache = cleanSearchCache || oldJsResort != newJsResort;sb.setConfig(SwitchboardConstants.SEARCH_JS_RESORT, newJsResort);-+final boolean oldStrictContentDom = sb.getConfigBool(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT);final boolean newStrictContentDom = post.getBoolean(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM);/* When this setting has changed we must clean up the search event cache as it affects how search results are retrieved */cleanSearchCache = cleanSearchCache || oldStrictContentDom != newStrictContentDom;sb.setConfig(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, newStrictContentDom);-+sb.setConfig(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,post.getBoolean(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED));sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, post.getBoolean(SwitchboardConstants.GREEDYLEARNING_ACTIVE));-+final boolean storeresult = post.getBoolean(SwitchboardConstants.REMOTESEARCH_RESULT_STORE);sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, storeresult);sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, post.getLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1));@@ -116,11 +118,11 @@ public class ConfigPortal_p {sb.setConfig("about.headline", post.get("about.headline", ""));sb.setConfig("about.body", post.get("about.body", ""));- String excludehosts = post.get("search.excludehosts", "");+ final String excludehosts = post.get("search.excludehosts", "");sb.setConfig("search.excludehosts", excludehosts);try {sb.setConfig("search.excludehosth", DigestURL.hosthashes(excludehosts));- } catch (MalformedURLException e) {+ } catch (final MalformedURLException e) {ConcurrentLog.logException(e);sb.setConfig("search.excludehosth", "");}@@ -143,23 +145,23 @@ public class ConfigPortal_p {sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, config.getProperty(SwitchboardConstants.PUBLIC_SEARCHPAGE,"true"));sb.setConfig("search.navigation", config.getProperty("search.navigation","hosts,authors,namespace,topics"));sb.setConfig("search.options", config.getProperty("search.options","true"));-+final boolean oldJsResort = sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT);final boolean newJsResort = Boolean.parseBoolean(config.getProperty(SwitchboardConstants.SEARCH_JS_RESORT, String.valueOf(SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT)));/* When this setting has changed we must clean up the search event cache as it affects how search results are retrieved */cleanSearchCache = cleanSearchCache || oldJsResort != newJsResort;sb.setConfig(SwitchboardConstants.SEARCH_JS_RESORT, newJsResort);-+final boolean oldStrictContentDom = sb.getConfigBool(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT);final boolean newStrictContentDom = Boolean.parseBoolean(config.getProperty(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, String.valueOf(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT)));/* When this setting has changed we must clean up the search event cache as it affects how search results are retrieved */cleanSearchCache = cleanSearchCache || oldStrictContentDom != newStrictContentDom;sb.setConfig(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, newStrictContentDom);-+sb.setConfig(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,Boolean.parseBoolean(config.getProperty(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,String.valueOf(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT))));-+sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, config.getProperty(SwitchboardConstants.GREEDYLEARNING_ACTIVE));sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE));sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE));@@ -170,12 +172,12 @@ public class ConfigPortal_p {sb.setConfig("search.excludehosts", config.getProperty("search.excludehosts",""));sb.setConfig("search.excludehosth", config.getProperty("search.excludehosth",""));}-+if(cleanSearchCache) {SearchEventCache.cleanupEvents(true);}}-+/* Acquire a transaction token for the next POST form submission */prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));@@ -192,11 +194,11 @@ public class ConfigPortal_p {prop.put(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM,sb.getConfigBool(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM,SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT) ? 1 : 0);-+prop.put(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,sb.getConfigBool(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT) ? 1 : 0);-+final boolean textSnippetsStatisticsEnabled = sb.getConfigBool(SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED,SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED_DEFAULT);@@ -204,15 +206,15 @@ public class ConfigPortal_p {prop.put(SwitchboardConstants.GREEDYLEARNING_ACTIVE, sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ACTIVE, false) ? 1 : 0);prop.put(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, sb.getConfig(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, "0"));-+prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, sb.getConfigBool(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, true) ? 1 : 0);- long resultStoredMaxSize = sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1);+ final long resultStoredMaxSize = sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1);if(resultStoredMaxSize > 0) {prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, resultStoredMaxSize);} else {prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, "");}-+/* Provide some basic stats about text snippets generation time to help choosing snippet options */if(textSnippetsStatisticsEnabled) {final long totalSnippets = TextSnippet.statistics.getTotalSnippets();@@ -290,7 +292,7 @@ public class ConfigPortal_p {*/private static String formatDuration(final long durationValue) {final Duration duration = Duration.ofMillis(durationValue);-+final String formattedDuration;if(duration.getSeconds() > 0) {formattedDuration = duration.getSeconds() + "s";diff --git a/htroot/ConfigProfile_p.java b/source/net/yacy/htroot/ConfigProfile_p.javasimilarity index 97%rename from htroot/ConfigProfile_p.javarename to source/net/yacy/htroot/ConfigProfile_p.javaindex 658046448..60f97f5d0 100644--- a/htroot/ConfigProfile_p.java+++ b/source/net/yacy/htroot/ConfigProfile_p.java@@ -29,6 +29,8 @@// javac -classpath .:../classes EditProfile_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import java.io.FileInputStream;import java.io.FileOutputStream;diff --git a/htroot/ConfigProperties_p.java b/source/net/yacy/htroot/ConfigProperties_p.javasimilarity index 96%rename from htroot/ConfigProperties_p.javarename to source/net/yacy/htroot/ConfigProperties_p.javaindex 4474b8ddb..b30a786b9 100644--- a/htroot/ConfigProperties_p.java+++ b/source/net/yacy/htroot/ConfigProperties_p.java@@ -28,6 +28,8 @@// javac -classpath .:../classes Config_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.ArrayList;import java.util.Collections;import java.util.Iterator;@@ -43,7 +45,7 @@ public class ConfigProperties_p {public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {// return variable that accumulates replacementsfinal serverObjects prop = new serverObjects();-+/* Acquire a transaction token for the next POST form submission */prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));@@ -54,7 +56,7 @@ public class ConfigProperties_p {if (post != null && post.containsKey("key") && post.containsKey("value")) {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);-+key = post.get("key").trim();value = post.get("value").trim();if (key != null && !key.isEmpty()) {diff --git a/htroot/ConfigRobotsTxt_p.java b/source/net/yacy/htroot/ConfigRobotsTxt_p.javasimilarity index 98%rename from htroot/ConfigRobotsTxt_p.javarename to source/net/yacy/htroot/ConfigRobotsTxt_p.javaindex bf140e8c2..eb559f6fd 100644--- a/htroot/ConfigRobotsTxt_p.java+++ b/source/net/yacy/htroot/ConfigRobotsTxt_p.java@@ -29,6 +29,8 @@// javac -classpath .:../classes ConfigRobotsTxt_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.search.Switchboard;import net.yacy.search.SwitchboardConstants;@@ -45,7 +47,7 @@ public class ConfigRobotsTxt_p {final RobotsTxtConfig rbc = sb.robotstxtConfig;// globaly overwitten by template engine (removed to avoid risk of NPE in intranet mode)- // prop.put("clientname", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()));+ // prop.put("clientname", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()));if (post != null) {if (post.containsKey("save")) {diff --git a/htroot/ConfigSearchBox.java b/source/net/yacy/htroot/ConfigSearchBox.javasimilarity index 95%rename from htroot/ConfigSearchBox.javarename to source/net/yacy/htroot/ConfigSearchBox.javaindex 05bde9ac3..8c125b6db 100644--- a/htroot/ConfigSearchBox.java+++ b/source/net/yacy/htroot/ConfigSearchBox.java@@ -23,6 +23,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.http.servlets.YaCyDefaultServlet;import net.yacy.search.Switchboard;diff --git a/htroot/ConfigSearchPage_p.java b/source/net/yacy/htroot/ConfigSearchPage_p.javasimilarity index 94%rename from htroot/ConfigSearchPage_p.javarename to source/net/yacy/htroot/ConfigSearchPage_p.javaindex 09967e60f..adebd8cf9 100644--- a/htroot/ConfigSearchPage_p.java+++ b/source/net/yacy/htroot/ConfigSearchPage_p.java@@ -25,6 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.sql.Date;import java.util.HashSet;import java.util.Map;@@ -56,21 +58,21 @@ public class ConfigSearchPage_p {if (post != null) {/* Check this is a valid transaction */TransactionManager.checkPostTransaction(header, post);-+final int initialNavMaxCOunt = sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT);final String initialNavConf = sb.getConfig("search.navigation", "");-+if (post.containsKey("searchpage_set")) {final String newGreeting = post.get(SwitchboardConstants.GREETING, "");// store this call as api callsb.tables.recordAPICall(post, "ConfigPortal_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);sb.setConfig("publicTopmenu", post.getBoolean("publicTopmenu"));-+sb.setConfig(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN,post.getBoolean(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN));-+sb.setConfig("search.options", post.getBoolean("search.options"));sb.setConfig("search.text", post.getBoolean("search.text"));@@ -81,14 +83,14 @@ public class ConfigSearchPage_p {sb.setConfig(SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON, post.getBoolean(SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON));sb.setConfig(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS, post.getBoolean(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS));-+// maximum number of initially displayed keywords/tags- int keywordsFirstMaxCount = post.getInt(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT,+ final int keywordsFirstMaxCount = post.getInt(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT,SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT_DEFAULT);if (keywordsFirstMaxCount > 0) {sb.setConfig(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT, keywordsFirstMaxCount);}-+sb.setConfig("search.result.show.date", post.getBoolean("search.result.show.date"));sb.setConfig("search.result.show.size", post.getBoolean("search.result.show.size"));sb.setConfig("search.result.show.metadata", post.getBoolean("search.result.show.metadata"));@@ -101,7 +103,7 @@ public class ConfigSearchPage_p {sb.setConfig("search.result.show.snapshots", post.getBoolean("search.result.show.snapshots"));// construct navigation String- Set<String> navConfigs = new HashSet<>();+ final Set<String> navConfigs = new HashSet<>();if (post.getBoolean("search.navigation.location")) {navConfigs.add("location");}@@ -115,7 +117,7 @@ public class ConfigSearchPage_p {navConfigs.add("date");}// append active navigator plugins- String[] activeNavNames = post.getAll("search.navigation.active");+ final String[] activeNavNames = post.getAll("search.navigation.active");for (final String navName : activeNavNames) {String navConfig = navName;final String navSortConfig = post.get("search.navigation." + navName + ".navSort");@@ -129,13 +131,13 @@ public class ConfigSearchPage_p {}sb.setConfig("search.navigation", navConfigs);// maxcount nav entries, default- int navmaxcnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT);+ final int navmaxcnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT);if (navmaxcnt > 5) {sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, navmaxcnt);}-+// maxcount dates navigator entries- int datesNavMaxCnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,+ final int datesNavMaxCnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT);if (datesNavMaxCnt > 5) {sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, datesNavMaxCnt);@@ -143,7 +145,7 @@ public class ConfigSearchPage_p {}if (post.containsKey("add.nav")) { // button: add navigator plugin to ative list- String navname = post.get("search.navigation.navname");+ final String navname = post.get("search.navigation.navname");if (navname != null && !navname.isEmpty()) {final Set<String> navConfigs = sb.getConfigSet("search.navigation");navConfigs.add(navname);@@ -196,14 +198,14 @@ public class ConfigSearchPage_p {config.getProperty(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,String.valueOf(QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT)));}-+if(!initialNavConf.equals(sb.getConfig("search.navigation", "")) || initialNavMaxCOunt != sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT)) {/* Clean up search events cache when necessary */SearchEventCache.cleanupEvents(true);}}-+/* Acquire a transaction token for the next POST form submission */prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));@@ -214,11 +216,11 @@ public class ConfigSearchPage_p {prop.putHTML(SwitchboardConstants.GREETING_IMAGE_ALT, sb.getConfig(SwitchboardConstants.GREETING_IMAGE_ALT, ""));prop.putHTML(SwitchboardConstants.INDEX_FORWARD, sb.getConfig(SwitchboardConstants.INDEX_FORWARD, ""));prop.put("publicTopmenu", sb.getConfigBool("publicTopmenu", false) ? 1 : 0);-+prop.put(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN,sb.getConfigBool(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN,SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN_DEFAULT) ? 1 : 0);-+prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0);prop.put("search.text", sb.getConfigBool("search.text", false) ? 1 : 0);@@ -226,7 +228,7 @@ public class ConfigSearchPage_p {prop.put("search.audio", sb.getConfigBool("search.audio", false) ? 1 : 0);prop.put("search.video", sb.getConfigBool("search.video", false) ? 1 : 0);prop.put("search.app", sb.getConfigBool("search.app", false) ? 1 : 0);-+prop.put(SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON,sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON,SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON_DEFAULT));@@ -234,11 +236,11 @@ public class ConfigSearchPage_p {prop.put(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS,sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS,SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS_DEFAULT) ? 1 : 0);-+prop.put(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT,sb.getConfigInt(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT,SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT_DEFAULT));-+prop.put("search.result.show.date", sb.getConfigBool("search.result.show.date", false) ? 1 : 0);prop.put("search.result.show.size", sb.getConfigBool("search.result.show.size", false) ? 1 : 0);prop.put("search.result.show.metadata", sb.getConfigBool("search.result.show.metadata", false) ? 1 : 0);@@ -274,10 +276,10 @@ public class ConfigSearchPage_p {prop.put("search.navigation.topics", topicsNavEnabled);prop.put("search.navigation.date", dateNavEnabled);// list active navigator plugins- Map<String, Navigator> navplugins = NavigatorPlugins.initFromCfgStrings(navConfigs);+ final Map<String, Navigator> navplugins = NavigatorPlugins.initFromCfgStrings(navConfigs);int i = 0;for (final String navname : navplugins.keySet()) {- Navigator nav = navplugins.get(navname);+ final Navigator nav = navplugins.get(navname);prop.put("search.navigation.plugin_" + i + "_name", navname);prop.put("search.navigation.plugin_" + i + "_displayname", nav.getDisplayName());final int navSort;@@ -286,16 +288,16 @@ public class ConfigSearchPage_p {} else {if(nav.getSort().getSortType() == NavigatorSortType.COUNT) {if(nav.getSort().getSortDir() == NavigatorSortDirection.DESC) {- navSort = 0;+ navSort = 0;} else {navSort = 1;}} else {if(nav.getSort().getSortDir() == NavigatorSortDirection.DESC) {- navSort = 2;+ navSort = 2;} else {navSort = 3;- }+ }}}prop.put("search.navigation.plugin_" + i + "_navSort", navSort);@@ -304,9 +306,9 @@ public class ConfigSearchPage_p {prop.put("search.navigation.plugin", i);// fill select field options (only navs not already active)- Map<String, String> defaultnavplugins = NavigatorPlugins.listAvailable();+ final Map<String, String> defaultnavplugins = NavigatorPlugins.listAvailable();i=0;- for (String navname : defaultnavplugins.keySet()) {+ for (final String navname : defaultnavplugins.keySet()) {if (!navplugins.containsKey(navname)) {prop.put("search.navigation.list_" + i + "_name", navname);prop.put("search.navigation.list_" + i + "_displayname", defaultnavplugins.get(navname));@@ -322,7 +324,7 @@ public class ConfigSearchPage_p {prop.put(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));-+prop.put(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));diff --git a/htroot/ConfigUpdate_p.java b/source/net/yacy/htroot/ConfigUpdate_p.javasimilarity index 99%rename from htroot/ConfigUpdate_p.javarename to source/net/yacy/htroot/ConfigUpdate_p.javaindex 9bc35e483..506d9e7f0 100644--- a/htroot/ConfigUpdate_p.java+++ b/source/net/yacy/htroot/ConfigUpdate_p.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.util.ArrayList;@@ -87,7 +89,7 @@ public class ConfigUpdate_p {break;}}- File downloadedRelease = versionToDownload.downloadRelease();+ final File downloadedRelease = versionToDownload.downloadRelease();if(downloadedRelease == null) {prop.put("candeploy_downloadError", "1");prop.putHTML("candeploy_downloadError_releasedownload", release);diff --git a/htroot/ConfigUser_p.java b/source/net/yacy/htroot/ConfigUser_p.javasimilarity index 93%rename from htroot/ConfigUser_p.javarename to source/net/yacy/htroot/ConfigUser_p.javaindex a3c4a950b..b567b5c6b 100644--- a/htroot/ConfigUser_p.java+++ b/source/net/yacy/htroot/ConfigUser_p.java@@ -18,6 +18,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.EnumMap;import java.util.Map;@@ -72,7 +74,7 @@ public class ConfigUser_p {//do nothing} else if (post.containsKey("user") && !"newuser".equals(post.get("user"))) {- UserDB.Entry entry = sb.userDB.getEntry(post.get("user"));+ final UserDB.Entry entry = sb.userDB.getEntry(post.get("user"));if (entry != null) {//TODO: set username read-only in htmlprop.putHTML("current_user", entry.getUserName());@@ -117,7 +119,7 @@ public class ConfigUser_p {rightsSet.put(right, post.containsKey(right.toString()) && "on".equals(post.get(right.toString())) ? "true" : "false");}- UserDB.Entry entry = sb.userDB.getEntry(username);+ final UserDB.Entry entry = sb.userDB.getEntry(username);if (entry != null) {try {@@ -151,7 +153,7 @@ public class ConfigUser_p {} catch (final Exception e) {ConcurrentLog.logException(e);}- YaCyHttpServer jhttpserver = (YaCyHttpServer) sb.getHttpServer();+ final YaCyHttpServer jhttpserver = (YaCyHttpServer) sb.getHttpServer();jhttpserver.resetUser(entry.getUserName());} else {prop.put("error", "1");@@ -162,7 +164,7 @@ public class ConfigUser_p {prop.putHTML("username", username);} else if (post.containsKey("delete")) {sb.userDB.removeEntry(post.get("username"));- YaCyHttpServer jhttpserver = (YaCyHttpServer) sb.getHttpServer();+ final YaCyHttpServer jhttpserver = (YaCyHttpServer) sb.getHttpServer();jhttpserver.removeUser(post.get("username"));prop.put(serverObjects.ACTION_LOCATION, "ConfigAccountList_p.html"); // jump back to user list}diff --git a/htroot/Connections_p.java b/source/net/yacy/htroot/Connections_p.javasimilarity index 96%rename from htroot/Connections_p.javarename to source/net/yacy/htroot/Connections_p.javaindex 5d28da486..759c2ba24 100644--- a/htroot/Connections_p.java+++ b/source/net/yacy/htroot/Connections_p.java@@ -27,6 +27,8 @@//javac -classpath .:../classes Network.java//if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.Collections;import java.util.LinkedList;import java.util.List;diff --git a/htroot/ContentAnalysis_p.java b/source/net/yacy/htroot/ContentAnalysis_p.javasimilarity index 99%rename from htroot/ContentAnalysis_p.javarename to source/net/yacy/htroot/ContentAnalysis_p.javaindex eed8455e7..a6972cd37 100644--- a/htroot/ContentAnalysis_p.java+++ b/source/net/yacy/htroot/ContentAnalysis_p.java@@ -18,6 +18,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import net.yacy.cora.federate.solr.Ranking;import net.yacy.cora.protocol.RequestHeader;import net.yacy.search.Switchboard;diff --git a/htroot/ContentIntegrationPHPBB3_p.java b/source/net/yacy/htroot/ContentIntegrationPHPBB3_p.javasimilarity index 82%rename from htroot/ContentIntegrationPHPBB3_p.javarename to source/net/yacy/htroot/ContentIntegrationPHPBB3_p.javaindex 0f0cc441b..0c4c52cd2 100644--- a/htroot/ContentIntegrationPHPBB3_p.java+++ b/source/net/yacy/htroot/ContentIntegrationPHPBB3_p.java@@ -22,6 +22,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.File;import net.yacy.cora.date.GenericFormatter;@@ -46,16 +48,16 @@ public class ContentIntegrationPHPBB3_p {if (post != null) {- String urlstub = post.get("content.phpbb3.urlstub", "");- String dbtype = post.get("content.phpbb3.dbtype", "");- String dbhost = post.get("content.phpbb3.dbhost", "");- int dbport = post.getInt("content.phpbb3.dbport", 3306);- String dbname = post.get("content.phpbb3.dbname", "");- String prefix = post.get("content.phpbb3.tableprefix", "");- String dbuser = post.get("content.phpbb3.dbuser", "");- String dbpw = post.get("content.phpbb3.dbpw", "");- int ppf = post.getInt("content.phpbb3.ppf", 1000);- String dumpfile = post.get("content.phpbb3.dumpfile", "");+ final String urlstub = post.get("content.phpbb3.urlstub", "");+ final String dbtype = post.get("content.phpbb3.dbtype", "");+ final String dbhost = post.get("content.phpbb3.dbhost", "");+ final int dbport = post.getInt("content.phpbb3.dbport", 3306);+ final String dbname = post.get("content.phpbb3.dbname", "");+ final String prefix = post.get("content.phpbb3.tableprefix", "");+ final String dbuser = post.get("content.phpbb3.dbuser", "");+ final String dbpw = post.get("content.phpbb3.dbpw", "");+ final int ppf = post.getInt("content.phpbb3.ppf", 1000);+ final String dumpfile = post.get("content.phpbb3.dumpfile", "");sb.setConfig("content.phpbb3.urlstub", urlstub);@@ -71,7 +73,7 @@ public class ContentIntegrationPHPBB3_p {if (post.containsKey("check")) {try {- Dao db = new PhpBB3Dao(+ final Dao db = new PhpBB3Dao(urlstub,dbtype,dbhost,@@ -95,7 +97,7 @@ public class ContentIntegrationPHPBB3_p {if (post.containsKey("export")) {try {- Dao db = new PhpBB3Dao(+ final Dao db = new PhpBB3Dao(urlstub,dbtype,dbhost,@@ -106,7 +108,7 @@ public class ContentIntegrationPHPBB3_p {dbpw);- int files = db.writeSurrogates(db.query(0, -1, 100), sb.surrogatesInPath, "fullexport-" + GenericFormatter.SHORT_SECOND_FORMATTER.format(), ppf);+ final int files = db.writeSurrogates(db.query(0, -1, 100), sb.surrogatesInPath, "fullexport-" + GenericFormatter.SHORT_SECOND_FORMATTER.format(), ppf);prop.put("export", 1);prop.put("export_files", files);db.close();@@ -118,12 +120,12 @@ public class ContentIntegrationPHPBB3_p {}if (post.containsKey("import")) {- File f = new File(dumpfile);+ final File f = new File(dumpfile);if (!f.exists()) {prop.put("import", 2);prop.put("import_error", "file " + dumpfile + " does not exist");} else try {- ImportDump importer = new ImportDump(+ final ImportDump importer = new ImportDump(dbtype,dbhost,dbport,diff --git a/htroot/CookieMonitorIncoming_p.java b/source/net/yacy/htroot/CookieMonitorIncoming_p.javasimilarity index 96%rename from htroot/CookieMonitorIncoming_p.javarename to source/net/yacy/htroot/CookieMonitorIncoming_p.javaindex ebdbbd850..76923822e 100644--- a/htroot/CookieMonitorIncoming_p.java+++ b/source/net/yacy/htroot/CookieMonitorIncoming_p.java@@ -27,6 +27,8 @@// javac -classpath .:../classes Network.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.Date;import java.util.Iterator;import java.util.Map;diff --git a/htroot/CookieMonitorOutgoing_p.java b/source/net/yacy/htroot/CookieMonitorOutgoing_p.javasimilarity index 96%rename from htroot/CookieMonitorOutgoing_p.javarename to source/net/yacy/htroot/CookieMonitorOutgoing_p.javaindex 23f77e217..d8e155468 100644--- a/htroot/CookieMonitorOutgoing_p.java+++ b/source/net/yacy/htroot/CookieMonitorOutgoing_p.java@@ -27,9 +27,12 @@// javac -classpath .:../classes Network.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.Date;import java.util.Iterator;import java.util.Map;+import javax.servlet.http.Cookie;import net.yacy.cora.protocol.HeaderFramework;diff --git a/htroot/CookieTest_p.java b/source/net/yacy/htroot/CookieTest_p.javasimilarity index 92%rename from htroot/CookieTest_p.javarename to source/net/yacy/htroot/CookieTest_p.javaindex 891692b4d..782fe8333 100644--- a/htroot/CookieTest_p.java+++ b/source/net/yacy/htroot/CookieTest_p.java@@ -27,7 +27,10 @@// javac -classpath .:../classes index.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import javax.servlet.http.Cookie;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.cora.protocol.ResponseHeader;import net.yacy.server.serverObjects;@@ -50,7 +53,7 @@ public class CookieTest_p {final servletProperties prop = new servletProperties();if (post.containsKey("act") && "clear_cookie".equals(post.get("act"))) {final ResponseHeader outgoingHeader = new ResponseHeader(200);- Cookie[] cookies = header.getCookies();+ final Cookie[] cookies = header.getCookies();if (cookies != null) {for (final Cookie cookie : cookies) {outgoingHeader.setCookie(cookie.getName(), cookie.getValue(), cookie.getMaxAge(), cookie.getPath(), cookie.getDomain(), cookie.getSecure());@@ -71,10 +74,10 @@ public class CookieTest_p {prop.putHTML("cookiesin_0_value", cookieValue);}- Cookie[] cookielst = header.getCookies();+ final Cookie[] cookielst = header.getCookies();int i = 0;if (cookielst != null) {- for (Cookie singleco : cookielst) {+ for (final Cookie singleco : cookielst) {prop.putHTML("cookiesout_" + i + "_string", singleco.getName() + "=" + singleco.getValue() + ";"); // output with ";" for compatiblity with cookiesini++;}diff --git a/htroot/CrawlCheck_p.java b/source/net/yacy/htroot/CrawlCheck_p.javasimilarity index 86%rename from htroot/CrawlCheck_p.javarename to source/net/yacy/htroot/CrawlCheck_p.javaindex 29b51f836..8a55dd466 100644--- a/htroot/CrawlCheck_p.java+++ b/source/net/yacy/htroot/CrawlCheck_p.java@@ -18,6 +18,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.net.MalformedURLException;import java.util.Collection;import java.util.LinkedHashSet;@@ -41,23 +43,23 @@ public class CrawlCheck_p {final serverObjects prop = new serverObjects();prop.put("starturls", "");if (post == null) return prop;-+if (post.containsKey("crawlcheck")) {-+// get the list of rootURls for this crawl start- Set<DigestURL> rootURLs = new LinkedHashSet<DigestURL>();- String crawlingStart0 = post.get("crawlingURLs","").trim();- String[] rootURLs0 = crawlingStart0.indexOf('\n') > 0 || crawlingStart0.indexOf('\r') > 0 ? crawlingStart0.split("[\\r\\n]+") : crawlingStart0.split(Pattern.quote("|"));+ final Set<DigestURL> rootURLs = new LinkedHashSet<DigestURL>();+ final String crawlingStart0 = post.get("crawlingURLs","").trim();+ final String[] rootURLs0 = crawlingStart0.indexOf('\n') > 0 || crawlingStart0.indexOf('\r') > 0 ? crawlingStart0.split("[\\r\\n]+") : crawlingStart0.split(Pattern.quote("|"));for (String crawlingStart: rootURLs0) {if (crawlingStart == null || crawlingStart.length() == 0) continue;// add the prefix http:// if necessary- int pos = crawlingStart.indexOf("://",0);+ final int pos = crawlingStart.indexOf("://",0);if (pos == -1) {if (crawlingStart.startsWith("www")) crawlingStart = "http://" + crawlingStart;if (crawlingStart.startsWith("ftp")) crawlingStart = "ftp://" + crawlingStart;}try {- DigestURL crawlingStartURL = new DigestURL(crawlingStart);+ final DigestURL crawlingStartURL = new DigestURL(crawlingStart);rootURLs.add(crawlingStartURL);} catch (final MalformedURLException e) {ConcurrentLog.logException(e);@@ -68,18 +70,18 @@ public class CrawlCheck_p {prop.put("table", 0);} else {prop.put("table", 1);-+// mass checkfinal ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));- Collection<CheckEntry> out = sb.robots.massCrawlCheck(rootURLs, agent);+ final Collection<CheckEntry> out = sb.robots.massCrawlCheck(rootURLs, agent);// evaluate the result from the concurrent computation-+// make a string that is used to fill the starturls field again// and analyze the urls to make the table rows- StringBuilder s = new StringBuilder(300);+ final StringBuilder s = new StringBuilder(300);int row = 0;- for (CheckEntry entry: out) {- String u = entry.digestURL.toNormalform(true);+ for (final CheckEntry entry: out) {+ final String u = entry.digestURL.toNormalform(true);s.append(u).append('\n');prop.put("table_list_" + row + "_url", u);@@ -95,7 +97,7 @@ public class CrawlCheck_p {prop.put("table_list_" + row + "_crawldelay", Math.max(agent.minimumDelta, entry.robotsTxtEntry.getCrawlDelayMillis()) + " ms");prop.put("table_list_" + row + "_sitemap", entry.robotsTxtEntry.getSitemaps().toString());}-+// try to load the urlif (robotsAllowed) {if (entry.response == null) {@@ -111,14 +113,14 @@ public class CrawlCheck_p {prop.put("table_list_" + row + "_access", "not loaded - prevented by robots.txt");}row++;-+}prop.put("table_list", row);prop.put("starturls", s.toString());-+}}-+return prop;}diff --git a/htroot/CrawlMonitorRemoteStart.java b/source/net/yacy/htroot/CrawlMonitorRemoteStart.javasimilarity index 98%rename from htroot/CrawlMonitorRemoteStart.javarename to source/net/yacy/htroot/CrawlMonitorRemoteStart.javaindex 906de0ea5..f41802ba9 100644--- a/htroot/CrawlMonitorRemoteStart.java+++ b/source/net/yacy/htroot/CrawlMonitorRemoteStart.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Iterator;import net.yacy.cora.protocol.RequestHeader;@@ -41,7 +43,7 @@ public class CrawlMonitorRemoteStart {final Switchboard sb = (Switchboard) env;final serverObjects prop = new serverObjects();- boolean dark = true;+ final boolean dark = true;// create other peer crawl table using YaCyNewsIterator<NewsDB.Record> recordIterator = sb.peers.newsPool.recordIterator(NewsPool.INCOMING_DB);diff --git a/htroot/CrawlProfileEditor_p.java b/source/net/yacy/htroot/CrawlProfileEditor_p.javasimilarity index 96%rename from htroot/CrawlProfileEditor_p.javarename to source/net/yacy/htroot/CrawlProfileEditor_p.javaindex 4ca089ad2..5eee8b7c3 100644--- a/htroot/CrawlProfileEditor_p.java+++ b/source/net/yacy/htroot/CrawlProfileEditor_p.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Map;import java.util.TreeMap;@@ -100,7 +102,7 @@ public class CrawlProfileEditor_p {if ((post != null) && (selentry != null)) {if (post.containsKey("submit")) {try {- for (CrawlProfile.CrawlAttribute attribute: CrawlProfile.CrawlAttribute.values()) {+ for (final CrawlProfile.CrawlAttribute attribute: CrawlProfile.CrawlAttribute.values()) {final String cval = selentry.get(attribute.key);final String val = (attribute.type == CrawlProfile.CrawlAttribute.BOOLEAN) ? Boolean.toString(post.containsKey(attribute.key)) : post.get(attribute.key, cval);if (!cval.equals(val)) {@@ -149,7 +151,7 @@ public class CrawlProfileEditor_p {prop.put("edit_name", selentry.collectionName());prop.put("edit_handle", selentry.handle());count = 0;- for (CrawlProfile.CrawlAttribute attribute: CrawlProfile.CrawlAttribute.values()) {+ for (final CrawlProfile.CrawlAttribute attribute: CrawlProfile.CrawlAttribute.values()) {final String val = selentry.get(attribute.key);prop.put(EDIT_ENTRIES_PREFIX + count + "_readonly", attribute.readonly ? "1" : "0");prop.put(EDIT_ENTRIES_PREFIX + count + "_readonly_name", attribute.key);diff --git a/htroot/CrawlResults.java b/source/net/yacy/htroot/CrawlResults.javasimilarity index 94%rename from htroot/CrawlResults.javarename to source/net/yacy/htroot/CrawlResults.javaindex 262ed172f..2c10be06a 100644--- a/htroot/CrawlResults.java+++ b/source/net/yacy/htroot/CrawlResults.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.time.ZoneId;import java.time.format.DateTimeFormatter;import java.util.Arrays;@@ -60,17 +62,17 @@ public class CrawlResults {/** Used for logging. */private static final String APP_NAME = "PLASMA";-+/** Default maximum lines number of the indexed table */private static final int DEFAULT_MAXIMUM_LINES = 500;-+public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env) {// return variable that accumulates replacementsfinal Switchboard sb = (Switchboard) env;final serverObjects prop = new serverObjects();int lines = DEFAULT_MAXIMUM_LINES;- boolean showCollection = sb.index.fulltext().getDefaultConfiguration().isEmpty() || sb.index.fulltext().getDefaultConfiguration().contains(CollectionSchema.collection_sxt);+ final boolean showCollection = sb.index.fulltext().getDefaultConfiguration().isEmpty() || sb.index.fulltext().getDefaultConfiguration().contains(CollectionSchema.collection_sxt);boolean showInit = env.getConfigBool("IndexMonitorInit", false);boolean showExec = env.getConfigBool("IndexMonitorExec", false);boolean showDate = env.getConfigBool("IndexMonitorDate", true);@@ -119,7 +121,7 @@ public class CrawlResults {return prop;}}-+String selectedblacklist = post.get("selectedblacklist",Blacklist.defaultBlacklist(ListManager.listsPath));if (post != null) {// custom number of lines@@ -142,18 +144,18 @@ public class CrawlResults {final String domain = post.get("domain", null);if (domain != null) {selectedblacklist = post.get("blacklistname");- Set<String> hostnames = new HashSet<String>();+ final Set<String> hostnames = new HashSet<String>();hostnames.add(domain);sb.index.fulltext().deleteStaleDomainNames(hostnames, null);ResultURLs.deleteDomain(tabletype, domain);-+// handle addtoblacklistif (post.containsKey("delandaddtoblacklist")) {try {Switchboard.urlBlacklist.add(selectedblacklist, domain, ".*");- } catch (PunycodeException e) {+ } catch (final PunycodeException e) {ConcurrentLog.warn(APP_NAME, "Unable to add blacklist entry to blacklist " + selectedblacklist, e);-+}}}@@ -213,7 +215,7 @@ public class CrawlResults {while (i.hasNext() && cnt < lines) {entry = i.next();try {- byte[] urlhash = UTF8.getBytes(entry.getKey());+ final byte[] urlhash = UTF8.getBytes(entry.getKey());urle = sb.index.fulltext().getMetadata(urlhash);if (urle == null) {sb.index.fulltext().commit(true);@@ -331,7 +333,7 @@ public class CrawlResults {prop.put("table_domains", cnt);// load all blacklist files located in the directory- List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);+ final List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);int blacklistCount = 0;if (dirlist != null) {for (final String element : dirlist) {@@ -345,7 +347,7 @@ public class CrawlResults {blacklistCount++;}prop.put("table_blacklists", blacklistCount);- }+ }}prop.put("process", tabletype.getCode());@@ -356,7 +358,7 @@ public class CrawlResults {private static final DateTimeFormatter DAY_FORMATTER = DateTimeFormatter.ofPattern("uuuu/MM/dd", Locale.US).withZone(ZoneId.systemDefault());-+/*** @param date a date to render as a String* @return the date formatted using the DAY_FORMATTER pattern.diff --git a/htroot/CrawlStartExpert.java b/source/net/yacy/htroot/CrawlStartExpert.javasimilarity index 97%rename from htroot/CrawlStartExpert.javarename to source/net/yacy/htroot/CrawlStartExpert.javaindex 635b76177..8a90510f5 100644--- a/htroot/CrawlStartExpert.java+++ b/source/net/yacy/htroot/CrawlStartExpert.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.ArrayList;import java.util.Collection;import java.util.List;@@ -172,7 +174,7 @@ public class CrawlStartExpert {// Unlimited crawl depth for URLs matching withif (post != null && post.containsKey("crawlingDepthExtension")) {- prop.put("crawlingDepthExtension",+ prop.put("crawlingDepthExtension",post.get("crawlingDepthExtension", ""));} else {prop.put("crawlingDepthExtension", CrawlProfile.MATCH_NEVER_STRING);@@ -314,7 +316,7 @@ public class CrawlStartExpert {// Filter on Content of Document: must matchif (post != null && post.containsKey("indexcontentmustmatch")) {- prop.put("indexcontentmustmatch",+ prop.put("indexcontentmustmatch",post.get("indexcontentmustmatch", ""));} else {prop.put("indexcontentmustmatch", CrawlProfile.MATCH_ALL_STRING);@@ -562,7 +564,7 @@ public class CrawlStartExpert {}// ---------- Agent name- List<String> agentNames = new ArrayList<String>();+ final List<String> agentNames = new ArrayList<String>();if (sb.isIntranetMode()) {agentNames.add(ClientIdentification.yacyIntranetCrawlerAgentName);}@@ -576,7 +578,7 @@ public class CrawlStartExpert {}String defaultAgentName = agentNames.get(0);if (post != null && post.containsKey("agentName")) {- String agentName = post.get("agentName", sb.isIntranetMode() ? ClientIdentification.yacyIntranetCrawlerAgentName : ClientIdentification.yacyInternetCrawlerAgentName);+ final String agentName = post.get("agentName", sb.isIntranetMode() ? ClientIdentification.yacyIntranetCrawlerAgentName : ClientIdentification.yacyInternetCrawlerAgentName);if (agentNames.contains(agentName)) defaultAgentName = agentName;}for (int i = 0; i < agentNames.size(); i++) {@@ -588,21 +590,21 @@ public class CrawlStartExpert {// ---------- Ignore Class Nameif (post != null && post.containsKey("ignoreclassname")) {- prop.put("ignoreclassname",+ prop.put("ignoreclassname",post.get("ignoreclassname", ""));} else {prop.put("ignoreclassname", "");}// ---------- Enrich Vocabulary- Collection<Tagging> vocs = LibraryProvider.autotagging.getVocabularies();+ final Collection<Tagging> vocs = LibraryProvider.autotagging.getVocabularies();if (vocs.size() == 0) {prop.put("vocabularySelect", 0);} else {prop.put("vocabularySelect", 1);int count = 0;- for (Tagging v: vocs) {- String value = post == null ? "" : post.get("vocabulary_" + v.getName() + "_class", "");+ for (final Tagging v: vocs) {+ final String value = post == null ? "" : post.get("vocabulary_" + v.getName() + "_class", "");prop.put("vocabularySelect_vocabularyset_" + count + "_name", v.getName());prop.put("vocabularySelect_vocabularyset_" + count + "_value", value);count++;@@ -611,7 +613,7 @@ public class CrawlStartExpert {}// ---------- Snapshot generation- boolean wkhtmltopdfAvailable = Html2Image.wkhtmltopdfAvailable();+ final boolean wkhtmltopdfAvailable = Html2Image.wkhtmltopdfAvailable();//boolean convertAvailable = Html2Image.convertAvailable();prop.put("snapshotsMaxDepth", post == null ? "-1" : post.get("snapshotsMaxDepth", "-1"));prop.put("snapshotsMustnotmatch", post == null ? "" : post.get("snapshotsMustnotmatch", ""));@@ -660,7 +662,7 @@ public class CrawlStartExpert {}// Target collection- boolean collectionEnabled =+ final boolean collectionEnabled =sb.index.fulltext().getDefaultConfiguration().isEmpty() ||sb.index.fulltext().getDefaultConfiguration().contains(CollectionSchema.collection_sxt);diff --git a/htroot/CrawlStartScanner_p.java b/source/net/yacy/htroot/CrawlStartScanner_p.javasimilarity index 93%rename from htroot/CrawlStartScanner_p.javarename to source/net/yacy/htroot/CrawlStartScanner_p.javaindex ed4e050ba..d76cf08b3 100644--- a/htroot/CrawlStartScanner_p.java+++ b/source/net/yacy/htroot/CrawlStartScanner_p.java@@ -18,6 +18,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.io.IOException;import java.net.InetAddress;import java.net.MalformedURLException;@@ -79,9 +81,9 @@ public class CrawlStartScanner_p {listall = true;post.put("source", "hosts");}- String[] hosts0 = hostt.indexOf('\n') > 0 || hostt.indexOf('\r') > 0 ? hostt.split("[\\r\\n]+") : hostt.split(Pattern.quote(","));- Set<String> hostSet = new LinkedHashSet<String>();- for (String s: hosts0) if (s != null && s.length() > 0) hostSet.add(s);+ final String[] hosts0 = hostt.indexOf('\n') > 0 || hostt.indexOf('\r') > 0 ? hostt.split("[\\r\\n]+") : hostt.split(Pattern.quote(","));+ final Set<String> hostSet = new LinkedHashSet<String>();+ for (final String s: hosts0) if (s != null && s.length() > 0) hostSet.add(s);final Set<InetAddress> ips = Domains.myIntranetIPs();prop.put("intranethosts", ips.toString());@@ -116,7 +118,7 @@ public class CrawlStartScanner_p {}}}- String hos = ""; for (String s: hostSet) hos += s + "\n";+ String hos = ""; for (final String s: hostSet) hos += s + "\n";prop.put("scanhosts", hos.trim());if (listall) {@@ -126,7 +128,7 @@ public class CrawlStartScanner_p {hostscore = sb.index.fulltext().getDefaultConnector().getFacets(AbstractSolrConnector.CATCHALL_QUERY, 1000, CollectionSchema.host_s.getSolrFieldName()).get(CollectionSchema.host_s.getSolrFieldName());} catch (final IOException e) {}if (hostscore != null) {- for (String s: hostscore) hostSet.add(s);+ for (final String s: hostscore) hostSet.add(s);}}@@ -149,12 +151,12 @@ public class CrawlStartScanner_p {// start a scannerfinal Scanner scanner = new Scanner(CONCURRENT_RUNNER, timeout);- boolean scanhttp = "on".equals(post.get("scanhttp", ""));- int scanhttpport = post.getInt("scanhttpport", 80);- boolean scanhttps = "on".equals(post.get("scanhttps", ""));- int scanhttpsport = post.getInt("scanhttpsport", 443);- boolean scanftp = "on".equals(post.get("scanftp", ""));- boolean scansmb = "on".equals(post.get("scansmb", ""));+ final boolean scanhttp = "on".equals(post.get("scanhttp", ""));+ final int scanhttpport = post.getInt("scanhttpport", 80);+ final boolean scanhttps = "on".equals(post.get("scanhttps", ""));+ final int scanhttpsport = post.getInt("scanhttpsport", 443);+ final boolean scanftp = "on".equals(post.get("scanftp", ""));+ final boolean scansmb = "on".equals(post.get("scansmb", ""));// select host base to scanif ("hosts".equals(post.get("source", ""))) {@@ -166,7 +168,7 @@ public class CrawlStartScanner_p {final int p = host.indexOf('/', 0);if (p >= 0) host = host.substring(0, p);InetAddress ip;- Collection<InetAddress> scanbase = new ArrayList<InetAddress>();+ final Collection<InetAddress> scanbase = new ArrayList<InetAddress>();if (host.length() > 0) {ip = Domains.dnsResolve(host); if (ip != null) scanbase.add(ip);if (host.startsWith("ftp.") || host.startsWith("www.") ||diff --git a/htroot/CrawlStartSite.java b/source/net/yacy/htroot/CrawlStartSite.javasimilarity index 95%rename from htroot/CrawlStartSite.javarename to source/net/yacy/htroot/CrawlStartSite.javaindex 4152b3f04..b7b74cd26 100644--- a/htroot/CrawlStartSite.java+++ b/source/net/yacy/htroot/CrawlStartSite.java@@ -9,7 +9,7 @@// $LastChangedBy$//// LICENSE-//+//// This program is free software; you can redistribute it and/or modify// it under the terms of the GNU General Public License as published by// the Free Software Foundation; either version 2 of the License, or@@ -24,12 +24,14 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;public class CrawlStartSite {-+public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {return CrawlStartExpert.respond(header, post, env);}diff --git a/htroot/Crawler_p.java b/source/net/yacy/htroot/Crawler_p.javasimilarity index 99%rename from htroot/Crawler_p.javarename to source/net/yacy/htroot/Crawler_p.javaindex 740d057c1..490f963b7 100644--- a/htroot/Crawler_p.java+++ b/source/net/yacy/htroot/Crawler_p.java@@ -19,6 +19,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.File;import java.io.FileInputStream;import java.io.FileNotFoundException;@@ -416,7 +418,7 @@ public class Crawler_p {if ("smb.ftp".indexOf(u.getProtocol()) >= 0 || "pdf".equals(MultiProtocolURL.getFileExtension(u.getFileName()))) anysmbftporpdf = true;}sb.index.fulltext().remove(deleteIDs);- deleteIDs.forEach(urlhash -> {try {sb.index.loadTimeIndex().remove(urlhash.getBytes());} catch (IOException e) {}});+ deleteIDs.forEach(urlhash -> {try {sb.index.loadTimeIndex().remove(urlhash.getBytes());} catch (final IOException e) {}});sb.crawlQueues.removeHosts(hosthashes);sb.index.fulltext().commit(true);@@ -438,7 +440,7 @@ public class Crawler_p {String basepath = u.toNormalform(true);if (!basepath.endsWith("/")) {final int p = basepath.lastIndexOf("/"); if (p > 0) basepath = basepath.substring(0, p + 1);}final int count = sb.index.fulltext().remove(basepath, deleteageDate);- try {sb.index.loadTimeIndex().clear();} catch (IOException e) {}+ try {sb.index.loadTimeIndex().clear();} catch (final IOException e) {}if (count > 0) ConcurrentLog.info("Crawler_p", "deleted " + count + " documents for host " + u.getHost());}}@@ -868,7 +870,7 @@ public class Crawler_p {* @throws IOException* @throws FileNotFoundException*/- private static List<AnchorURL> crawlingFileStart(final File crawlingFile, int timezoneOffset,+ private static List<AnchorURL> crawlingFileStart(final File crawlingFile, final int timezoneOffset,final String crawlingFileContent) throws MalformedURLException, IOException, FileNotFoundException {List<AnchorURL> hyperlinks_from_file;// check if the crawl filter works correctlydiff --git a/htroot/DictionaryLoader_p.java b/source/net/yacy/htroot/DictionaryLoader_p.javasimilarity index 98%rename from htroot/DictionaryLoader_p.javarename to source/net/yacy/htroot/DictionaryLoader_p.javaindex 09098a7df..9b415d46b 100644--- a/htroot/DictionaryLoader_p.java+++ b/source/net/yacy/htroot/DictionaryLoader_p.java@@ -18,6 +18,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.File;import java.io.FileInputStream;import java.io.IOException;@@ -246,7 +248,7 @@ public class DictionaryLoader_p {prop.put("geo2ActionLoaded_error", e.getMessage());}}-+if (post.containsKey("geo0Remove")) {FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.file());FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.fileDisabled());@@ -259,7 +261,7 @@ public class DictionaryLoader_p {LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB0.nickname);prop.put("geo0ActionDeactivated", 1);}-+if (post.containsKey("geo1Remove")) {FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.file());FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.fileDisabled());@@ -336,7 +338,7 @@ public class DictionaryLoader_p {LibraryProvider.initDidYouMean();prop.put("drw0ActionActivated", 1);}-+if (post.containsKey("syn0Deactivate")) {synonym_de_production.delete();SynonymLibrary.init(synonyms_path);@@ -347,29 +349,29 @@ public class DictionaryLoader_p {try {inStream = new FileInputStream(synonym_de_default);FileUtils.copy(inStream, synonym_de_production);- } catch (IOException e) {+ } catch (final IOException e) {ConcurrentLog.logException(e);} finally {if(inStream != null) {try {inStream.close();- } catch (IOException ignored) {+ } catch (final IOException ignored) {ConcurrentLog.warn("DictionaryLoader", "Could not close file " + synonym_de_default.getName());}}}SynonymLibrary.init(synonyms_path);}-+if (post.containsKey("syn1Deactivate")) {synonym_en_production.delete();SynonymLibrary.init(synonyms_path);}-+if (post.containsKey("syn1Activate")) {try {FileUtils.copy(new FileInputStream(synonym_en_default), synonym_en_production);- } catch (IOException e) {+ } catch (final IOException e) {ConcurrentLog.logException(e);}SynonymLibrary.init(synonyms_path);@@ -383,22 +385,22 @@ public class DictionaryLoader_p {if (post.containsKey("syn2Activate")) {try {FileUtils.copy(new FileInputStream(synonym_ru_default), synonym_ru_production);- } catch (IOException e) {+ } catch (final IOException e) {ConcurrentLog.logException(e);}SynonymLibrary.init(synonyms_path);- }+ }if (post != null) {// check here if there is a possibility synonym libraries have been activated/deactivatedprop.put("syn0Status", synonym_de_production.exists() ? 1 : 0);prop.put("syn1Status", synonym_en_production.exists() ? 1 : 0);prop.put("syn2Status", synonym_ru_production.exists() ? 1 : 0);}-+// check status againboolean keepPlacesTagging = false;for (final LibraryProvider.Dictionary dictionary: LibraryProvider.Dictionary.values()) {- int newstatus = dictionary.file().exists() ? 1 : dictionary.fileDisabled().exists() ? 2 : 0;+ final int newstatus = dictionary.file().exists() ? 1 : dictionary.fileDisabled().exists() ? 2 : 0;if (newstatus == 1) keepPlacesTagging = true;prop.put(dictionary.nickname + "Status", newstatus);}@@ -407,7 +409,7 @@ public class DictionaryLoader_p {if (!keepPlacesTagging) {LibraryProvider.autotagging.removePlaces();}-+return prop; // return rewrite values for templates}}diff --git a/htroot/Help.java b/source/net/yacy/htroot/Help.javasimilarity index 92%rename from htroot/Help.javarename to source/net/yacy/htroot/Help.javaindex 875f36d2c..81d2aa03a 100644--- a/htroot/Help.java+++ b/source/net/yacy/htroot/Help.java@@ -4,6 +4,8 @@//$LastChangedBy$//+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;diff --git a/htroot/IndexBrowser_p.java b/source/net/yacy/htroot/IndexBrowser_p.javasimilarity index 76%rename from htroot/IndexBrowser_p.javarename to source/net/yacy/htroot/IndexBrowser_p.javaindex bf66476f0..0751912bd 100644--- a/htroot/IndexBrowser_p.java+++ b/source/net/yacy/htroot/IndexBrowser_p.java@@ -18,6 +18,10 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;++import static net.yacy.repository.BlacklistHelper.addBlacklistEntry;+import java.io.File;import java.io.IOException;import java.net.MalformedURLException;@@ -60,7 +64,6 @@ import net.yacy.crawler.retrieval.Request;import net.yacy.kelondro.data.meta.URIMetadataNode;import net.yacy.peers.graphics.WebStructureGraph.StructureEntry;import net.yacy.repository.Blacklist;-import static net.yacy.repository.BlacklistHelper.addBlacklistEntry;import net.yacy.search.Switchboard;import net.yacy.search.SwitchboardConstants;import net.yacy.search.index.Fulltext;@@ -125,7 +128,7 @@ public class IndexBrowser_p {public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {// return variable that accumulates replacementsfinal Switchboard sb = (Switchboard) env;- Fulltext fulltext = sb.index.fulltext();+ final Fulltext fulltext = sb.index.fulltext();final boolean autoload = sb.getConfigBool("browser.autoload", true);final boolean load4everyone = sb.getConfigBool("browser.load4everyone", false);final boolean loadRight = autoload || load4everyone; // add config later@@ -173,7 +176,7 @@ public class IndexBrowser_p {if (sb.crawlStacker.size() > 2) wait = false;try {url = new DigestURL(load);- String reasonString = sb.crawlStacker.stackCrawl(new Request(+ final String reasonString = sb.crawlStacker.stackCrawl(new Request(sb.peers.mySeed().hash.getBytes(),url, null, load, new Date(),sb.crawler.defaultProxyProfile.handle(),@@ -209,10 +212,10 @@ public class IndexBrowser_p {if (post.containsKey("hosts")) {// generate host listtry {- boolean onlyCrawling = "crawling".equals(post.get("hosts", ""));- boolean onlyErrors = "error".equals(post.get("hosts", ""));+ final boolean onlyCrawling = "crawling".equals(post.get("hosts", ""));+ final boolean onlyErrors = "error".equals(post.get("hosts", ""));- int maxcount = 2 * 3 * 2 * 5 * 7 * 2 * 3; // which makes nice matrixes for 2, 3, 4, 5, 6, 7, 8, 9 rows/colums+ final int maxcount = 2 * 3 * 2 * 5 * 7 * 2 * 3; // which makes nice matrixes for 2, 3, 4, 5, 6, 7, 8, 9 rows/colums// collect hosts from indexReversibleScoreMap<String> hostscore = fulltext.getDefaultConnector().getFacets(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", maxcount, CollectionSchema.host_s.getSolrFieldName()).get(CollectionSchema.host_s.getSolrFieldName());@@ -222,9 +225,9 @@ public class IndexBrowser_p {final Map<String, Integer[]> crawler = sb.crawlQueues.noticeURL.getDomainStackHosts(StackType.LOCAL, sb.robots);final Map<String, Integer> hostNameToPendingCount = new HashMap<>();- for(Entry<String, Integer[]>crawlerEntry: crawler.entrySet()) {+ for(final Entry<String, Integer[]>crawlerEntry: crawler.entrySet()) {/* The local stack returns keys composed of "hostname:port" : we now sum pending URLs counts by host name */- String hostName = Domains.stripToHostName(crawlerEntry.getKey());+ final String hostName = Domains.stripToHostName(crawlerEntry.getKey());Integer pendingCount = hostNameToPendingCount.get(hostName);if(pendingCount == null) {pendingCount = 0;@@ -234,21 +237,21 @@ public class IndexBrowser_p {}// collect the errorurls- Map<String, ReversibleScoreMap<String>> exclfacets = fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.excl.name(), maxcount, CollectionSchema.host_s.getSolrFieldName());- ReversibleScoreMap<String> exclscore = exclfacets == null ? new ClusteredScoreMap<String>(true) : exclfacets.get(CollectionSchema.host_s.getSolrFieldName());- Map<String, ReversibleScoreMap<String>> failfacets = fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.fail.name(), maxcount, CollectionSchema.host_s.getSolrFieldName());- ReversibleScoreMap<String> failscore = failfacets == null ? new ClusteredScoreMap<String>(true) : failfacets.get(CollectionSchema.host_s.getSolrFieldName());+ final Map<String, ReversibleScoreMap<String>> exclfacets = fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.excl.name(), maxcount, CollectionSchema.host_s.getSolrFieldName());+ final ReversibleScoreMap<String> exclscore = exclfacets == null ? new ClusteredScoreMap<String>(true) : exclfacets.get(CollectionSchema.host_s.getSolrFieldName());+ final Map<String, ReversibleScoreMap<String>> failfacets = fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.fail.name(), maxcount, CollectionSchema.host_s.getSolrFieldName());+ final ReversibleScoreMap<String> failscore = failfacets == null ? new ClusteredScoreMap<String>(true) : failfacets.get(CollectionSchema.host_s.getSolrFieldName());int c = 0;- Iterator<String> i = hostscore.keys(false);+ final Iterator<String> i = hostscore.keys(false);String host;while (i.hasNext() && c < maxcount) {host = i.next();prop.putHTML("hosts_list_" + c + "_host", host);- boolean inCrawler = hostNameToPendingCount.containsKey(host);- int exclcount = exclscore.get(host);- int failcount = failscore.get(host);- int errors = exclcount + failcount;+ final boolean inCrawler = hostNameToPendingCount.containsKey(host);+ final int exclcount = exclscore.get(host);+ final int failcount = failscore.get(host);+ final int errors = exclcount + failcount;prop.put("hosts_list_" + c + "_count", hostscore.get(host));prop.put("hosts_list_" + c + "_crawler", inCrawler ? 1 : 0);if (inCrawler) {@@ -284,33 +287,33 @@ public class IndexBrowser_p {// write host analysis if path after host is emptyif (uri.getPath().length() <= 1 && host != null && host.length() > 0 && sb.getConfigBool("decoration.hostanalysis", false)) {//how many documents per crawldepth_i; get crawldepth_i facet for host- ArrayList<String> ff = new ArrayList<>();- for (CollectionSchema csf: CollectionSchema.values()) {+ final ArrayList<String> ff = new ArrayList<>();+ for (final CollectionSchema csf: CollectionSchema.values()) {if ((csf.getType() != SolrType.num_integer && csf.getType() != SolrType.num_long) || csf.isMultiValued()) continue;- String facetfield = csf.getSolrFieldName();+ final String facetfield = csf.getSolrFieldName();if (!fulltext.getDefaultConfiguration().contains(facetfield)) continue;ff.add(csf.getSolrFieldName());}// add also vocabulary counters- Map<String, ReversibleScoreMap<String>> vocabularyFacet = sb.index.fulltext().getDefaultConnector().getFacets(CollectionSchema.vocabularies_sxt.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM, 100, CollectionSchema.vocabularies_sxt.getSolrFieldName());+ final Map<String, ReversibleScoreMap<String>> vocabularyFacet = sb.index.fulltext().getDefaultConnector().getFacets(CollectionSchema.vocabularies_sxt.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM, 100, CollectionSchema.vocabularies_sxt.getSolrFieldName());if (vocabularyFacet.size() > 0) {- Collection<String> vocnames = vocabularyFacet.values().iterator().next().keyList(true);- for (String vocname: vocnames) {+ final Collection<String> vocnames = vocabularyFacet.values().iterator().next().keyList(true);+ for (final String vocname: vocnames) {ff.add(CollectionSchema.VOCABULARY_PREFIX + vocname + CollectionSchema.VOCABULARY_LOGCOUNT_SUFFIX);ff.add(CollectionSchema.VOCABULARY_PREFIX + vocname + CollectionSchema.VOCABULARY_LOGCOUNTS_SUFFIX);}}// list the facets- String[] facetfields = ff.toArray(new String[ff.size()]);- Map<String, ReversibleScoreMap<String>> facets = fulltext.getDefaultConnector().getFacets(CollectionSchema.host_s.getSolrFieldName() + ":\"" + host + "\"", 100, facetfields);+ final String[] facetfields = ff.toArray(new String[ff.size()]);+ final Map<String, ReversibleScoreMap<String>> facets = fulltext.getDefaultConnector().getFacets(CollectionSchema.host_s.getSolrFieldName() + ":\"" + host + "\"", 100, facetfields);int fc = 0;- for (Map.Entry<String, ReversibleScoreMap<String>> facetentry: facets.entrySet()) {- ReversibleScoreMap<String> facetfieldmap = facetentry.getValue();+ for (final Map.Entry<String, ReversibleScoreMap<String>> facetentry: facets.entrySet()) {+ final ReversibleScoreMap<String> facetfieldmap = facetentry.getValue();if (facetfieldmap.size() == 0) continue;- TreeMap<Long, Integer> statMap = new TreeMap<>();- for (String k: facetfieldmap) statMap.put(Long.parseLong(k), facetfieldmap.get(k));+ final TreeMap<Long, Integer> statMap = new TreeMap<>();+ for (final String k: facetfieldmap) statMap.put(Long.parseLong(k), facetfieldmap.get(k));prop.put("hostanalysis_facets_" + fc + "_facetname", facetentry.getKey());- int c = 0; for (Entry<Long, Integer> entry: statMap.entrySet()) {+ int c = 0; for (final Entry<Long, Integer> entry: statMap.entrySet()) {prop.put("hostanalysis_facets_" + fc + "_facet_" + c + "_key", entry.getKey());prop.put("hostanalysis_facets_" + fc + "_facet_" + c + "_count", entry.getValue());prop.put("hostanalysis_facets_" + fc + "_facet_" + c + "_a", "solr/collection1/select?q=host_s:" + host + " AND " + facetentry.getKey() + ":" + entry.getKey() + "&defType=edismax&start=0&rows=1000&fl=sku,crawldepth_i");@@ -334,8 +337,8 @@ public class IndexBrowser_p {// try to re-load all urls that have load errors and matches with this prefix.reload404 = true;}- int facetcount = post.getInt("facetcount", 0);- boolean complete = post.getBoolean("complete");+ final int facetcount = post.getInt("facetcount", 0);+ final boolean complete = post.getBoolean("complete");if (complete) { // we want only root paths for complete listsp = path.indexOf('/', 10);if (p > 0) path = path.substring(0, p + 1);@@ -352,11 +355,11 @@ public class IndexBrowser_p {// generate file list from pathprop.putHTML("outbound_host", host);prop.putHTML("inbound_host", host);- String hosthash = uri.hosthash();- String[] pathparts = uri.getPaths();+ final String hosthash = uri.hosthash();+ final String[] pathparts = uri.getPaths();// get all files for a specific host from the index- StringBuilder q = new StringBuilder();+ final StringBuilder q = new StringBuilder();if (host == null) {if (path.startsWith("file://")) {q.append(CollectionSchema.url_protocol_s.getSolrFieldName()).append(":file");@@ -365,7 +368,7 @@ public class IndexBrowser_p {q.append(CollectionSchema.host_s.getSolrFieldName()).append(":\"").append(host).append("\"");}if (pathparts.length > 0 && pathparts[0].length() > 0) {- for (String pe: pathparts) {+ for (final String pe: pathparts) {if (pe.length() > 0) q.append(" AND ").append(CollectionSchema.url_paths_sxt.getSolrFieldName()).append(":\"").append(pe).append('\"');}} else {@@ -373,10 +376,10 @@ public class IndexBrowser_p {q.append(" AND ").append(CollectionSchema.url_paths_sxt.getSolrFieldName()).append(AbstractSolrConnector.CATCHALL_DTERM);}}-+// if reload errors is requested, limit results to pages with some failreason setif(reload404) q.append(" AND ").append(CollectionSchema.failtype_s.getSolrFieldName()).append(AbstractSolrConnector.CATCHALL_DTERM);-+final int pageSize = 100;final BlockingQueue<SolrDocument> docs = new ArrayBlockingQueue<>(pageSize);final List<String> queries = new ArrayList<>();@@ -397,26 +400,26 @@ public class IndexBrowser_p {CollectionSchema.references_exthosts_i.getSolrFieldName()));solrQueryTask.start();- Set<String> storedDocs = new HashSet<String>();- Map<String, FailType> errorDocs = new HashMap<String, FailType>();- Set<String> inboundLinks = new HashSet<String>();- Map<String, ReversibleScoreMap<String>> outboundHosts = new HashMap<String, ReversibleScoreMap<String>>();- Map<String, InfoCacheEntry> infoCache = new HashMap<String, InfoCacheEntry>();+ final Set<String> storedDocs = new HashSet<String>();+ final Map<String, FailType> errorDocs = new HashMap<String, FailType>();+ final Set<String> inboundLinks = new HashSet<String>();+ final Map<String, ReversibleScoreMap<String>> outboundHosts = new HashMap<String, ReversibleScoreMap<String>>();+ final Map<String, InfoCacheEntry> infoCache = new HashMap<String, InfoCacheEntry>();int hostsize = 0;final List<String> deleteIDs = new ArrayList<String>();final Collection<String> reloadURLs = new ArrayList<String>();final Set<String> reloadURLCollection = new HashSet<String>();- long timeoutList = System.currentTimeMillis() + TIMEOUT;+ final long timeoutList = System.currentTimeMillis() + TIMEOUT;long remainingTime = TIMEOUT;- long timeoutReferences = System.currentTimeMillis() + TIMEOUT;- ReferenceReportCache rrCache = sb.index.getReferenceReportCache();+ final long timeoutReferences = System.currentTimeMillis() + TIMEOUT;+ final ReferenceReportCache rrCache = sb.index.getReferenceReportCache();try {SolrDocument doc = docs.poll(remainingTime, TimeUnit.MILLISECONDS);while (doc != AbstractSolrConnector.POISON_DOCUMENT && doc != null) {String u = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());- String errortype = (String) doc.getFieldValue(CollectionSchema.failtype_s.getSolrFieldName());- FailType error = errortype == null ? null : FailType.valueOf(errortype);- String ids = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName());+ final String errortype = (String) doc.getFieldValue(CollectionSchema.failtype_s.getSolrFieldName());+ final FailType error = errortype == null ? null : FailType.valueOf(errortype);+ final String ids = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName());infoCache.put(ids, new InfoCacheEntry(sb.index.fulltext(), rrCache, doc, ids, System.currentTimeMillis() < timeoutReferences));if (u.startsWith(path)) {if (delete) {@@ -424,7 +427,7 @@ public class IndexBrowser_p {} else {if (error == null) storedDocs.add(u); else {if (reload404 && error == FailType.fail) {- ArrayList<String> collections = (ArrayList<String>) doc.getFieldValue(CollectionSchema.collection_sxt.getSolrFieldName());+ final ArrayList<String> collections = (ArrayList<String>) doc.getFieldValue(CollectionSchema.collection_sxt.getSolrFieldName());if (collections != null) reloadURLCollection.addAll(collections);reloadURLs.add(u);}@@ -451,7 +454,7 @@ public class IndexBrowser_p {while (links.hasNext()) {u = links.next();try {- MultiProtocolURL mu = new MultiProtocolURL(u);+ final MultiProtocolURL mu = new MultiProtocolURL(u);if (mu.getHost() != null) {ReversibleScoreMap<String> lks = outboundHosts.get(mu.getHost());if (lks == null) {@@ -477,47 +480,47 @@ public class IndexBrowser_p {if (deleteIDs.size() > 0) sb.remove(deleteIDs);if (reloadURLs.size() > 0) {final Map<String, Pattern> cm = new LinkedHashMap<String, Pattern>();- for (String collection: reloadURLCollection) cm.put(collection, QueryParams.catchall_pattern);+ for (final String collection: reloadURLCollection) cm.put(collection, QueryParams.catchall_pattern);sb.reload(reloadURLs, cm.size() > 0 ? cm : CrawlProfile.collectionParser("user"), false);-+}// collect from crawler- List<Request> domainStackReferences = sb.crawlQueues.noticeURL.getDomainStackReferences(StackType.LOCAL, host, 1000, 3000);- Set<String> loadingLinks = new HashSet<String>();- for (Request crawlEntry: domainStackReferences) loadingLinks.add(crawlEntry.url().toNormalform(true));+ final List<Request> domainStackReferences = sb.crawlQueues.noticeURL.getDomainStackReferences(StackType.LOCAL, host, 1000, 3000);+ final Set<String> loadingLinks = new HashSet<String>();+ for (final Request crawlEntry: domainStackReferences) loadingLinks.add(crawlEntry.url().toNormalform(true));// now combine all lists into one- Map<String, StoreType> files = new HashMap<String, StoreType>();- for (String u: storedDocs) files.put(u, StoreType.INDEX);- for (Map.Entry<String, FailType> e: errorDocs.entrySet()) files.put(e.getKey(), e.getValue() == FailType.fail ? StoreType.FAILED : StoreType.EXCLUDED);- for (String u: inboundLinks) if (!files.containsKey(u)) files.put(u, StoreType.LINK);- if(!reload404) for (String u: loadingLinks) if (u.startsWith(path) && !files.containsKey(u)) files.put(u, StoreType.LINK);+ final Map<String, StoreType> files = new HashMap<String, StoreType>();+ for (final String u: storedDocs) files.put(u, StoreType.INDEX);+ for (final Map.Entry<String, FailType> e: errorDocs.entrySet()) files.put(e.getKey(), e.getValue() == FailType.fail ? StoreType.FAILED : StoreType.EXCLUDED);+ for (final String u: inboundLinks) if (!files.containsKey(u)) files.put(u, StoreType.LINK);+ if(!reload404) for (final String u: loadingLinks) if (u.startsWith(path) && !files.containsKey(u)) files.put(u, StoreType.LINK);ConcurrentLog.info("IndexBrowser_p", "collected " + files.size() + " urls for path " + path);// distinguish files and folders- Map<String, Object> list = new TreeMap<String, Object>(); // a directory list; if object is boolean, its a file; if its a int[], then its a folder- int pl = path.length();+ final Map<String, Object> list = new TreeMap<String, Object>(); // a directory list; if object is boolean, its a file; if its a int[], then its a folder+ final int pl = path.length();String file;- for (Map.Entry<String, StoreType> entry: files.entrySet()) {+ for (final Map.Entry<String, StoreType> entry: files.entrySet()) {if (entry.getKey().length() < pl) continue; // this is not inside the pathif (!entry.getKey().startsWith(path)) continue;file = entry.getKey().substring(pl);- StoreType type = entry.getValue();+ final StoreType type = entry.getValue();p = file.indexOf('/');if (p < 0) {// this is a filelist.put(entry.getKey(), type); // StoreType value: this is a file; true -> file is in index; false -> not in index, maybe in crawler} else {// this is a directory path or a file in a subdirectory- String remainingPath = file.substring(0, p + 1);+ final String remainingPath = file.substring(0, p + 1);if (complete && remainingPath.indexOf('.') > 0) {list.put(entry.getKey(), type); // StoreType value: this is a file} else {- String dir = path + remainingPath;- Object c = list.get(dir);+ final String dir = path + remainingPath;+ final Object c = list.get(dir);if (c == null) {- int[] linkedStoredIncrawlerError = new int[]{0,0,0,0,0};+ final int[] linkedStoredIncrawlerError = new int[]{0,0,0,0,0};if (type == StoreType.LINK) linkedStoredIncrawlerError[0]++;if (type == StoreType.INDEX) linkedStoredIncrawlerError[1]++;if (loadingLinks.contains(entry.getKey())) linkedStoredIncrawlerError[2]++;@@ -537,18 +540,18 @@ public class IndexBrowser_p {int c = 0;// first list only foldersint filecounter = 0;- for (Map.Entry<String, Object> entry: list.entrySet()) {+ for (final Map.Entry<String, Object> entry: list.entrySet()) {if ((entry.getValue() instanceof StoreType)) {filecounter++;} else {// this is a folderprop.put("files_list_" + c + "_type", 1);prop.putHTML("files_list_" + c + "_type_url", entry.getKey());- int linked = ((int[]) entry.getValue())[0];- int stored = ((int[]) entry.getValue())[1];- int crawler = ((int[]) entry.getValue())[2];- int excl = ((int[]) entry.getValue())[3];- int error = ((int[]) entry.getValue())[4];+ final int linked = ((int[]) entry.getValue())[0];+ final int stored = ((int[]) entry.getValue())[1];+ final int crawler = ((int[]) entry.getValue())[2];+ final int excl = ((int[]) entry.getValue())[3];+ final int error = ((int[]) entry.getValue())[4];prop.put("files_list_" + c + "_type_stored", stored);prop.put("files_list_" + c + "_type_linked", linked);prop.put("files_list_" + c + "_type_pendingVisible", crawler > 0 ? 1 : 0);@@ -561,34 +564,34 @@ public class IndexBrowser_p {}}// then list files- for (Map.Entry<String, Object> entry: list.entrySet()) {+ for (final Map.Entry<String, Object> entry: list.entrySet()) {if (entry.getValue() instanceof StoreType) {// this is a fileprop.put("files_list_" + c + "_type", 0);prop.putHTML("files_list_" + c + "_type_url", entry.getKey());- StoreType type = (StoreType) entry.getValue();+ final StoreType type = (StoreType) entry.getValue();try {uri = new DigestURL(entry.getKey());} catch (final MalformedURLException e) {uri = null;}- HarvestProcess process = uri == null ? null : sb.crawlQueues.exists(uri.hash()); // todo: cannot identify errors- boolean loading = load.equals(entry.getKey()) || (process != null && process != HarvestProcess.ERRORS);- boolean error = process == HarvestProcess.ERRORS || type == StoreType.EXCLUDED || type == StoreType.FAILED;- boolean dc = type != StoreType.INDEX && !error && !loading && list.containsKey(entry.getKey() + "/");+ final HarvestProcess process = uri == null ? null : sb.crawlQueues.exists(uri.hash()); // todo: cannot identify errors+ final boolean loading = load.equals(entry.getKey()) || (process != null && process != HarvestProcess.ERRORS);+ final boolean error = process == HarvestProcess.ERRORS || type == StoreType.EXCLUDED || type == StoreType.FAILED;+ final boolean dc = type != StoreType.INDEX && !error && !loading && list.containsKey(entry.getKey() + "/");if (!dc) {prop.put("files_list_" + c + "_type_stored", type == StoreType.INDEX ? 1 : error ? 3 : loading ? 2 : 0 /*linked*/);if (type == StoreType.INDEX) {- String ids = ASCII.String(uri.hash());- InfoCacheEntry ice = infoCache.get(ids);+ final String ids = ASCII.String(uri.hash());+ final InfoCacheEntry ice = infoCache.get(ids);prop.put("files_list_" + c + "_type_stored_comment", ice == null ? "" : ice.toString()); // ice.toString() contains html, therefore do not use putHTML here}prop.put("files_list_" + c + "_type_stored_load", loadRight ? 1 : 0);if (error) {- FailType failType = errorDocs.get(entry.getKey());+ final FailType failType = errorDocs.get(entry.getKey());if (failType == null) {// maybe this is only in the errorURL//Metadata faildoc = sb.index.fulltext().getDefaultConnector().getMetadata(ASCII.String(uri.hash()));prop.putHTML("files_list_" + c + "_type_stored_error", "unknown error");} else {- String ids = ASCII.String(uri.hash());- InfoCacheEntry ice = infoCache.get(ids);+ final String ids = ASCII.String(uri.hash());+ final InfoCacheEntry ice = infoCache.get(ids);prop.put("files_list_" + c + "_type_stored_error", failType == FailType.excl ? "excluded from indexing" : "load fail" + (ice == null ? "" : "; " + ice.toString()));}}@@ -614,13 +617,13 @@ public class IndexBrowser_p {prop.put("files_linkgraph_host", uri.getHost());// generate inbound-links table- StructureEntry struct = sb.webStructure.incomingReferences(hosthash);+ final StructureEntry struct = sb.webStructure.incomingReferences(hosthash);if (struct != null && struct.references.size() > 0) {maxcount = 200;- ReversibleScoreMap<String> score = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);- for (Map.Entry<String, Integer> entry: struct.references.entrySet()) score.set(entry.getKey(), entry.getValue());+ final ReversibleScoreMap<String> score = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);+ for (final Map.Entry<String, Integer> entry: struct.references.entrySet()) score.set(entry.getKey(), entry.getValue());c = 0;- Iterator<String> i = score.keys(false);+ final Iterator<String> i = score.keys(false);while (i.hasNext() && c < maxcount) {host = i.next();prop.putHTML("inbound_list_" + c + "_host", sb.webStructure.hostHash2hostName(host));@@ -636,10 +639,10 @@ public class IndexBrowser_p {// generate outbound-links tableif (outboundHosts.size() > 0) {maxcount = 200;- ReversibleScoreMap<String> score = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);- for (Map.Entry<String, ReversibleScoreMap<String>> entry: outboundHosts.entrySet()) score.set(entry.getKey(), entry.getValue().size());+ final ReversibleScoreMap<String> score = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);+ for (final Map.Entry<String, ReversibleScoreMap<String>> entry: outboundHosts.entrySet()) score.set(entry.getKey(), entry.getValue().size());c = 0;- Iterator<String> i = score.keys(false);+ final Iterator<String> i = score.keys(false);while (i.hasNext() && c < maxcount) {host = i.next();prop.putHTML("outbound_list_" + c + "_host", host);@@ -657,10 +660,10 @@ public class IndexBrowser_p {ConcurrentLog.logException(e);}- // for addtoblacklist button+ // for addtoblacklist buttonif (post.containsKey("addtoblacklist") && sb != null) {final File blacklistsPath = sb.getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT);- String blacklistname = Blacklist.defaultBlacklist(blacklistsPath);+ final String blacklistname = Blacklist.defaultBlacklist(blacklistsPath);if (blacklistname != null) {String hoststr = pathURI.getHost();if (hoststr.startsWith("www.")) { // to allow the automatic matching for www.host.com and host.com@@ -668,7 +671,7 @@ public class IndexBrowser_p {}final boolean isok = addBlacklistEntry(blacklistname,- hoststr + "/.*"); // as yacy expands host pattern to match www.host.com as well as host.com no need to prefix with "*."+ hoststr + "/.*"); // as yacy expands host pattern to match www.host.com as well as host.com no need to prefix with "*."if (isok) {prop.put("files_blkadded", 1); // glyphicon-ok on (display)} else {@@ -690,14 +693,14 @@ public class IndexBrowser_p {public Double cr_c;public int crawldepth, references, references_internal, references_external, references_exthosts;public List<String> references_internal_urls, references_external_urls;- public InfoCacheEntry(final Fulltext fulltext, final ReferenceReportCache rrCache, final SolrDocument doc, final String urlhash, boolean fetchReferences) {+ public InfoCacheEntry(final Fulltext fulltext, final ReferenceReportCache rrCache, final SolrDocument doc, final String urlhash, final boolean fetchReferences) {this.cr_c = (Double) doc.getFieldValue(CollectionSchema.cr_host_chance_d.getSolrFieldName());this.cr_n = (Integer) doc.getFieldValue(CollectionSchema.cr_host_norm_i.getSolrFieldName());- Integer cr = (Integer) doc.getFieldValue(CollectionSchema.crawldepth_i.getSolrFieldName());- Integer rc = (Integer) doc.getFieldValue(CollectionSchema.references_i.getSolrFieldName());- Integer rc_internal = (Integer) doc.getFieldValue(CollectionSchema.references_internal_i.getSolrFieldName());- Integer rc_external = (Integer) doc.getFieldValue(CollectionSchema.references_external_i.getSolrFieldName());- Integer rc_exthosts = (Integer) doc.getFieldValue(CollectionSchema.references_exthosts_i.getSolrFieldName());+ final Integer cr = (Integer) doc.getFieldValue(CollectionSchema.crawldepth_i.getSolrFieldName());+ final Integer rc = (Integer) doc.getFieldValue(CollectionSchema.references_i.getSolrFieldName());+ final Integer rc_internal = (Integer) doc.getFieldValue(CollectionSchema.references_internal_i.getSolrFieldName());+ final Integer rc_external = (Integer) doc.getFieldValue(CollectionSchema.references_external_i.getSolrFieldName());+ final Integer rc_exthosts = (Integer) doc.getFieldValue(CollectionSchema.references_exthosts_i.getSolrFieldName());this.crawldepth = (cr == null || cr.intValue() < 0) ? 0 : cr.intValue(); // for lazy value storage; non-existent means: stored as '0'this.references = (rc == null || rc.intValue() <= 0) ? 0 : rc.intValue();this.references_internal = (rc_internal == null || rc_internal.intValue() <= 0) ? 0 : rc_internal.intValue();@@ -707,22 +710,22 @@ public class IndexBrowser_p {if (fetchReferences) {// get the references from the citation indextry {- ReferenceReport rr = rrCache.getReferenceReport(urlhash, false);- List<String> internalIDs = new ArrayList<String>();- List<String> externalIDs = new ArrayList<String>();- HandleSet iids = rr.getInternallIDs();- for (byte[] b: iids) internalIDs.add(ASCII.String(b));- HandleSet eids = rr.getExternalIDs();- for (byte[] b: eids) externalIDs.add(ASCII.String(b));+ final ReferenceReport rr = rrCache.getReferenceReport(urlhash, false);+ final List<String> internalIDs = new ArrayList<String>();+ final List<String> externalIDs = new ArrayList<String>();+ final HandleSet iids = rr.getInternallIDs();+ for (final byte[] b: iids) internalIDs.add(ASCII.String(b));+ final HandleSet eids = rr.getExternalIDs();+ for (final byte[] b: eids) externalIDs.add(ASCII.String(b));// get all urls from the index and store them here- for (String id: internalIDs) {+ for (final String id: internalIDs) {if (id.equals(urlhash)) continue; // no self-references- String u = fulltext.getURL(id);+ final String u = fulltext.getURL(id);if (u != null) this.references_internal_urls.add(u);}- for (String id: externalIDs) {+ for (final String id: externalIDs) {if (id.equals(urlhash)) continue; // no self-references- String u = fulltext.getURL(id);+ final String u = fulltext.getURL(id);if (u != null) this.references_external_urls.add(u);}} catch (final IOException e) {@@ -733,17 +736,17 @@ public class IndexBrowser_p {}@Overridepublic String toString() {- StringBuilder sbi = new StringBuilder();+ final StringBuilder sbi = new StringBuilder();int c = 0;- for (String s: this.references_internal_urls) {+ for (final String s: this.references_internal_urls) {sbi.append("<a href='").append(s).append("' target='_blank'><img src='env/grafics/i16.gif' alt='info' title='" + s + "' width='12' height='12'/></a>");c++;if (c % 80 == 0) sbi.append("<br/>");}if (sbi.length() > 0) sbi.insert(0, "<br/>internal referrer:");- StringBuilder sbe = new StringBuilder();+ final StringBuilder sbe = new StringBuilder();c = 0;- for (String s: this.references_external_urls) {+ for (final String s: this.references_external_urls) {sbe.append("<a href='").append(s).append("' target='_blank'><img src='env/grafics/i16.gif' alt='info' title='" + s + "' width='12' height='12'/></a>");c++;if (c % 80 == 0) sbe.append("<br/>");diff --git a/htroot/IndexControlRWIs_p.java b/source/net/yacy/htroot/IndexControlRWIs_p.javasimilarity index 98%rename from htroot/IndexControlRWIs_p.javarename to source/net/yacy/htroot/IndexControlRWIs_p.javaindex a66b9fe27..ec5b03f28 100644--- a/htroot/IndexControlRWIs_p.java+++ b/source/net/yacy/htroot/IndexControlRWIs_p.java@@ -25,6 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.IOException;import java.util.ArrayList;import java.util.Arrays;@@ -107,10 +109,10 @@ public class IndexControlRWIs_p {// clean up all search eventsSearchEventCache.cleanupEvents(true);- Segment segment = sb.index;+ final Segment segment = sb.index;if ( post != null ) {- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));final String keystring = post.get("keystring", "").trim();byte[] keyhash = post.get("keyhash", "").trim().getBytes();if (keystring.length() > 0 && !keystring.contains(errmsg)) {@@ -304,7 +306,7 @@ public class IndexControlRWIs_p {Reference iEntry;while (urlIter.hasNext()) {iEntry = urlIter.next();- boolean exists = segment.fulltext().exists(ASCII.String(iEntry.urlhash()));+ final boolean exists = segment.fulltext().exists(ASCII.String(iEntry.urlhash()));if (exists) {try {knownURLs.put(iEntry.urlhash());@@ -404,13 +406,13 @@ public class IndexControlRWIs_p {ConcurrentLog.logException(e);}try {- String u = segment.fulltext().getURL(ASCII.String(b));+ final String u = segment.fulltext().getURL(ASCII.String(b));url = u == null ? null : new DigestURL(u);segment.fulltext().remove(b);if ( url != null ) {items.add(new BlacklistHostAndPath(url.getHost(), url.getFile()));}- } catch (IOException e1) {+ } catch (final IOException e1) {ConcurrentLog.logException(e1);}}@@ -421,7 +423,7 @@ public class IndexControlRWIs_p {try {Switchboard.urlBlacklist.add(supportedBlacklistType,blacklist, items);- } catch (PunycodeException e) {+ } catch (final PunycodeException e) {ConcurrentLog.warn(APP_NAME,"Unable to add blacklist entries to blacklist "+ supportedBlacklistType, e);@@ -440,7 +442,7 @@ public class IndexControlRWIs_p {ConcurrentLog.logException(e);}try {- String u = segment.fulltext().getURL(ASCII.String(b));+ final String u = segment.fulltext().getURL(ASCII.String(b));url = u == null ? null : new DigestURL(u);segment.fulltext().remove(b);if ( url != null ) {@@ -460,7 +462,7 @@ public class IndexControlRWIs_p {}}}- } catch (IOException e1) {+ } catch (final IOException e1) {ConcurrentLog.logException(e1);}}diff --git a/htroot/IndexControlURLs_p.java b/source/net/yacy/htroot/IndexControlURLs_p.javasimilarity index 92%rename from htroot/IndexControlURLs_p.javarename to source/net/yacy/htroot/IndexControlURLs_p.javaindex 6772c414d..72594e912 100644--- a/htroot/IndexControlURLs_p.java+++ b/source/net/yacy/htroot/IndexControlURLs_p.java@@ -21,6 +21,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.net.MalformedURLException;@@ -63,8 +65,8 @@ public class IndexControlURLs_p {final String nextTransactionToken = TransactionManager.getTransactionToken(header);prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken);- Segment segment = sb.index;- long ucount = segment.fulltext().collectionSize();+ final Segment segment = sb.index;+ final long ucount = segment.fulltext().collectionSize();// set default valuesprop.put("urlstring", "");@@ -79,7 +81,7 @@ public class IndexControlURLs_p {prop.put("reload", 0);prop.put("dumprestore", 1);prop.put("dumprestore_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken);- List<File> dumpFiles = segment.fulltext().dumpFiles();+ final List<File> dumpFiles = segment.fulltext().dumpFiles();prop.put("dumprestore_dumpfile", dumpFiles.size() == 0 ? "" : dumpFiles.get(dumpFiles.size() - 1).getAbsolutePath());prop.put("dumprestore_optimizemax", 10);prop.put("dumprestore_rebootSolrEnabled",@@ -163,9 +165,9 @@ public class IndexControlURLs_p {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));- int i = segment.removeAllUrlReferences(urlhash.getBytes(), sb.loader, agent, CacheStrategy.IFEXIST);- try {segment.loadTimeIndex().remove(urlhash.getBytes());} catch (IOException e) {}+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));+ final int i = segment.removeAllUrlReferences(urlhash.getBytes(), sb.loader, agent, CacheStrategy.IFEXIST);+ try {segment.loadTimeIndex().remove(urlhash.getBytes());} catch (final IOException e) {}prop.put("result", "Deleted URL and " + i + " references from " + i + " word indexes.");}@@ -184,7 +186,7 @@ public class IndexControlURLs_p {prop.putHTML("result", "Removed URL " + url);}segment.loadTimeIndex().remove(urlhash.getBytes());- } catch (IOException e) {+ } catch (final IOException e) {prop.putHTML("result", "Error when querying the url hash " + urlhash + ":" + e.getMessage());}}@@ -202,7 +204,7 @@ public class IndexControlURLs_p {prop.put("result", "No input given; nothing deleted.");} else {sb.urlRemove(segment, urlhash.getBytes());- try {segment.loadTimeIndex().remove(urlhash.getBytes());} catch (IOException e) {}+ try {segment.loadTimeIndex().remove(urlhash.getBytes());} catch (final IOException e) {}prop.putHTML("result", "Removed URL " + urlstring);}}@@ -266,10 +268,10 @@ public class IndexControlURLs_p {TransactionManager.checkPostTransaction(header, post);final String domain = post.get("domain");- Set<String> hostnames = new HashSet<String>();+ final Set<String> hostnames = new HashSet<String>();hostnames.add(domain);segment.fulltext().deleteStaleDomainNames(hostnames, null);- try {segment.loadTimeIndex().clear();} catch (IOException e) {} // delete all to prevent that existing entries reject reloading+ try {segment.loadTimeIndex().clear();} catch (final IOException e) {} // delete all to prevent that existing entries reject reloading// trigger the loading of the tablepost.put("statistics", "");}@@ -280,9 +282,9 @@ public class IndexControlURLs_p {int cnt = 0;try {final Fulltext metadata = segment.fulltext();- Map<String, ReversibleScoreMap<String>> scores = metadata.getDefaultConnector().getFacets(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", count, CollectionSchema.host_s.getSolrFieldName());- ReversibleScoreMap<String> stats = scores.get(CollectionSchema.host_s.getSolrFieldName());- Iterator<String> statsiter = stats.keys(false);+ final Map<String, ReversibleScoreMap<String>> scores = metadata.getDefaultConnector().getFacets(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", count, CollectionSchema.host_s.getSolrFieldName());+ final ReversibleScoreMap<String> stats = scores.get(CollectionSchema.host_s.getSolrFieldName());+ final Iterator<String> statsiter = stats.keys(false);boolean dark = true;String hostname;prop.put("statisticslines_domains_" + cnt + "lines", count);diff --git a/htroot/IndexCreateLoaderQueue_p.java b/source/net/yacy/htroot/IndexCreateLoaderQueue_p.javasimilarity index 92%rename from htroot/IndexCreateLoaderQueue_p.javarename to source/net/yacy/htroot/IndexCreateLoaderQueue_p.javaindex 331e46cca..377107cab 100644--- a/htroot/IndexCreateLoaderQueue_p.java+++ b/source/net/yacy/htroot/IndexCreateLoaderQueue_p.java@@ -27,6 +27,8 @@// javac -classpath .:../classes IndexCreate_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.Map;import net.yacy.cora.document.encoding.ASCII;@@ -45,7 +47,7 @@ public class IndexCreateLoaderQueue_p {final Switchboard sb = (Switchboard) env;final serverObjects prop = new serverObjects();- Map<DigestURL, Request> map = sb.crawlQueues.activeWorkerEntries();+ final Map<DigestURL, Request> map = sb.crawlQueues.activeWorkerEntries();if (map.size() == 0) {prop.put("loader-set", "0");} else {@@ -53,7 +55,7 @@ public class IndexCreateLoaderQueue_p {boolean dark = true;Seed initiator;int count = 0;- for (Request element : map.values()) {+ for (final Request element : map.values()) {if (element == null) continue;initiator = sb.peers.getConnected((element.initiator() == null) ? "" : ASCII.String(element.initiator()));diff --git a/htroot/IndexCreateParserErrors_p.java b/source/net/yacy/htroot/IndexCreateParserErrors_p.javasimilarity index 93%rename from htroot/IndexCreateParserErrors_p.javarename to source/net/yacy/htroot/IndexCreateParserErrors_p.javaindex 11dc9d075..0840be2e0 100644--- a/htroot/IndexCreateParserErrors_p.java+++ b/source/net/yacy/htroot/IndexCreateParserErrors_p.java@@ -23,6 +23,8 @@// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.ArrayList;import net.yacy.cora.date.GenericFormatter;@@ -72,18 +74,18 @@ public class IndexCreateParserErrors_p {dark = true;DigestURL url;int j=0;- ArrayList<CollectionConfiguration.FailDoc> l = sb.crawlQueues.errorURL.list(showRejectedCount);+ final ArrayList<CollectionConfiguration.FailDoc> l = sb.crawlQueues.errorURL.list(showRejectedCount);CollectionConfiguration.FailDoc entry;for (int i = l.size() - 1; i >= 0; i--) {entry = l.get(i);if (entry == null) continue;url = entry.getDigestURL();if (url == null) continue;-+prop.putHTML("rejected_list_"+j+"_time", GenericFormatter.SIMPLE_FORMATTER.format(entry.getFailDate()));prop.putHTML("rejected_list_"+j+"_url", url.toNormalform(false));-- String cause = entry.getFailReason();++ final String cause = entry.getFailReason();if (cause == null) {// well that should not really happen but occurs in a specific combination of running crawls for same domains with different url filtersprop.put("rejected_list_"+j+"_failreason", "no fail reason given");diff --git a/htroot/IndexCreateQueues_p.java b/source/net/yacy/htroot/IndexCreateQueues_p.javasimilarity index 91%rename from htroot/IndexCreateQueues_p.javarename to source/net/yacy/htroot/IndexCreateQueues_p.javaindex 766a9e635..f4ae0e7e9 100644--- a/htroot/IndexCreateQueues_p.java+++ b/source/net/yacy/htroot/IndexCreateQueues_p.java@@ -1,22 +1,24 @@+package net.yacy.htroot;+import java.net.MalformedURLException;import java.time.ZoneId;import java.time.format.DateTimeFormatter;import java.util.ArrayList;+import java.util.Collections;import java.util.Date;import java.util.HashSet;import java.util.Iterator;+import java.util.LinkedHashMap;import java.util.List;import java.util.Locale;import java.util.Map;import java.util.Map.Entry;-import java.util.Collections;-import java.util.stream.Collectors;-import java.util.LinkedHashMap;-import java.util.TreeMap;import java.util.Set;+import java.util.TreeMap;import java.util.regex.Pattern;import java.util.regex.PatternSyntaxException;+import java.util.stream.Collectors;import org.apache.http.conn.util.InetAddressUtils;@@ -39,7 +41,7 @@ public class IndexCreateQueues_p {private static final DateTimeFormatter DAY_FORMATTER = DateTimeFormatter.ofPattern("uuuu/MM/dd", Locale.US).withZone(ZoneId.systemDefault());-+/*** @param date a date to render as a String* @return the date formatted using the DAY_FORMATTER pattern.@@ -76,7 +78,7 @@ public class IndexCreateQueues_p {if (post.containsKey("embed")) embed = true;if (post.containsKey("sort")) {- String countSort = post.get("sort");+ final String countSort = post.get("sort");if (countSort.equals("count")) sortByCount = +1;if (countSort.equals("-count")) sortByCount = -1;if (countSort.equals("host")) sortByHost = +1;@@ -110,16 +112,16 @@ public class IndexCreateQueues_p {int removedByHosts = 0;if (option == URL && deletepattern.startsWith(".*") && deletepattern.endsWith(".*")) {// try to delete that using the host name- Set<String> hosthashes = new HashSet<String>();- String hn = deletepattern.substring(2, deletepattern.length() - 2);+ final Set<String> hosthashes = new HashSet<String>();+ final String hn = deletepattern.substring(2, deletepattern.length() - 2);try {hosthashes.add(DigestURL.hosthash(hn, hn.startsWith("ftp") ? 21 : 80));hosthashes.add(DigestURL.hosthash(hn, 443));removedByHosts = sb.crawlQueues.removeHosts(hosthashes);- } catch (MalformedURLException e) {+ } catch (final MalformedURLException e) {}}-+if (removedByHosts == 0) {// iterating through the list of URLsfinal Iterator<Request> iter = sb.crawlQueues.noticeURL.iterator(stackType);@@ -128,7 +130,7 @@ public class IndexCreateQueues_p {while (iter.hasNext()) {if ((entry = iter.next()) == null) continue;String value = null;-+location: switch (option) {case URL: value = (entry.url() == null) ? null : entry.url().toString(); break location;case ANCHOR: value = entry.name(); break location;@@ -139,7 +141,7 @@ public class IndexCreateQueues_p {case MODIFIED: value = daydate(entry.appdate()); break location;default: value = null; break location;}-+if (value != null && compiledPattern.matcher(value).matches()) removehashes.add(entry.url().hash());}ConcurrentLog.info("IndexCreateQueues_p", "created a remove list with " + removehashes.size() + " entries for pattern '" + deletepattern + "'");@@ -155,7 +157,7 @@ public class IndexCreateQueues_p {}}- int stackSize = sb.crawlQueues.noticeURL.stackSize(stackType);+ final int stackSize = sb.crawlQueues.noticeURL.stackSize(stackType);if (stackSize == 0) {prop.put("crawler", "0");} else {@@ -182,41 +184,41 @@ public class IndexCreateQueues_p {prop.put("crawler_showtable_sortedByCount",true);if (sortByCount < 0) {prop.put("crawler_showtable_sortedByCount_asc", false);- Map<String, Integer[]> hosts_sorted = hosts.entrySet().stream()+ final Map<String, Integer[]> hosts_sorted = hosts.entrySet().stream().sorted( (e1, e2)->e2.getValue()[0].compareTo(e1.getValue()[0]) ).collect(Collectors.toMap(Entry::getKey, Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new));hosts = hosts_sorted;}else {prop.put("crawler_showtable_sortedByCount_asc", true);- Map<String, Integer[]> hosts_sorted = hosts.entrySet().stream()+ final Map<String, Integer[]> hosts_sorted = hosts.entrySet().stream().sorted( (e1, e2)->e1.getValue()[0].compareTo(e2.getValue()[0]) ).collect(Collectors.toMap(Entry::getKey, Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new));hosts = hosts_sorted;}}-+if ( sortByHost!=0 ) {prop.put("crawler_showtable_sortedByCount",false);prop.put("crawler_showtable_sortedByCount_asc",true);prop.put("crawler_showtable_sortedByHost",true);if (sortByHost < 0) {prop.put("crawler_showtable_sortedByHost_asc", false);- Map<String, Integer[]> hosts_sorted = new TreeMap<String, Integer[]>(Collections.reverseOrder());+ final Map<String, Integer[]> hosts_sorted = new TreeMap<String, Integer[]>(Collections.reverseOrder());hosts_sorted.putAll(hosts);hosts = hosts_sorted;}else {prop.put("crawler_showtable_sortedByHost_asc", true);- Map<String, Integer[]> hosts_sorted = new TreeMap<String, Integer[]>(hosts);+ final Map<String, Integer[]> hosts_sorted = new TreeMap<String, Integer[]>(hosts);hosts = hosts_sorted;}}}int hc = 0;- for (Map.Entry<String, Integer[]> host: hosts.entrySet()) {- String hostnameport = host.getKey();+ for (final Map.Entry<String, Integer[]> host: hosts.entrySet()) {+ final String hostnameport = host.getKey();String hostname = Domains.stripToHostName(hostnameport);if(InetAddressUtils.isIPv6Address(hostname)) {hostname = "[" + hostname + "]"; // HostBalancer.getDomainStackReferences() function requires square brackets around IPV6 addresses@@ -228,13 +230,13 @@ public class IndexCreateQueues_p {prop.putHTML("crawler_host_" + hc + "_queuename", stackType.name());prop.put("crawler_host_" + hc + "_hostcount", host.getValue()[0]);prop.put("crawler_host_" + hc + "_hostdelta", host.getValue()[1] == Integer.MIN_VALUE ? "not accessed" : Integer.toString(host.getValue()[1]));- List<Request> domainStackReferences = sb.crawlQueues.noticeURL.getDomainStackReferences(stackType, hostname, urlsPerHost, 10000);+ final List<Request> domainStackReferences = sb.crawlQueues.noticeURL.getDomainStackReferences(stackType, hostname, urlsPerHost, 10000);Seed initiator;String profileHandle;CrawlProfile profileEntry;int count = 0;- for (Request request: domainStackReferences) {+ for (final Request request: domainStackReferences) {if (request == null) continue;initiator = sb.peers.getConnected(request.initiator() == null ? "" : ASCII.String(request.initiator()));profileHandle = request.profileHandle();diff --git a/htroot/IndexDeletion_p.java b/source/net/yacy/htroot/IndexDeletion_p.javasimilarity index 82%rename from htroot/IndexDeletion_p.javarename to source/net/yacy/htroot/IndexDeletion_p.javaindex 3b437a7d8..99aed1077 100644--- a/htroot/IndexDeletion_p.java+++ b/source/net/yacy/htroot/IndexDeletion_p.java@@ -18,6 +18,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.io.IOException;import java.net.MalformedURLException;import java.util.Date;@@ -56,8 +58,8 @@ public class IndexDeletion_p {/* Acquire a transaction token for the next POST form submission */prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));- SolrConnector defaultConnector = sb.index.fulltext().getDefaultConnector();- SolrConnector webgraphConnector = sb.index.fulltext().getWebgraphConnector();+ final SolrConnector defaultConnector = sb.index.fulltext().getDefaultConnector();+ final SolrConnector webgraphConnector = sb.index.fulltext().getWebgraphConnector();if (post == null || post.size() == 0) defaultConnector.commit(false); // we must do a commit here because the user cannot see a proper count.String schemaName = CollectionSchema.CORE_NAME;@@ -65,16 +67,16 @@ public class IndexDeletion_p {// Delete by URL MatchingString urldelete = post == null ? "" : post.get("urldelete", "");- boolean urldelete_mm_subpath_checked = post == null ? true : post.get("urldelete-mm", "subpath").equals("subpath");+ final boolean urldelete_mm_subpath_checked = post == null ? true : post.get("urldelete-mm", "subpath").equals("subpath");prop.putHTML("urldelete", urldelete);prop.put("urldelete-mm-subpath-checked", urldelete_mm_subpath_checked ? 1 : 0);prop.put("urldelete-mm-regexp-checked", urldelete_mm_subpath_checked ? 0 : 1);prop.put("urldelete-active", 0);// Delete by Age- int timedelete_number = post == null ? 14 : post.getInt("timedelete-number", 14);- String timedelete_unit = post == null ? "day" : post.get("timedelete-unit", "day");- boolean timedelete_source_loaddate_checked = post == null ? true : post.get("timedelete-source", "loaddate").equals("loaddate");+ final int timedelete_number = post == null ? 14 : post.getInt("timedelete-number", 14);+ final String timedelete_unit = post == null ? "day" : post.get("timedelete-unit", "day");+ final boolean timedelete_source_loaddate_checked = post == null ? true : post.get("timedelete-source", "loaddate").equals("loaddate");for (int i = 1; i <= 90; i++) prop.put("timedelete-n-" + i, 0);prop.put("timedelete-n-" + timedelete_number, timedelete_number);prop.put("timedelete-u-year", timedelete_unit.equals("year") ? 1 : 0);@@ -92,11 +94,11 @@ public class IndexDeletion_p {collectiondelete_mode_unassigned_checked = false;prop.put("collectiondelete-select", 1);try {- ScoreMap<String> collectionMap = defaultConnector.getFacets("*:*", 1000, CollectionSchema.collection_sxt.getSolrFieldName()).get(CollectionSchema.collection_sxt.getSolrFieldName());- Iterator<String> i = collectionMap.iterator();+ final ScoreMap<String> collectionMap = defaultConnector.getFacets("*:*", 1000, CollectionSchema.collection_sxt.getSolrFieldName()).get(CollectionSchema.collection_sxt.getSolrFieldName());+ final Iterator<String> i = collectionMap.iterator();int c = 0;while (i.hasNext()) {- String collection = i.next();+ final String collection = i.next();prop.put("collectiondelete-select_list_" + c + "_collection-name", collection + "/" + collectionMap.get(collection));prop.put("collectiondelete-select_list_" + c + "_collection-value", collection);c++;@@ -128,26 +130,26 @@ public class IndexDeletion_p {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);- boolean simulate = post.containsKey("simulate-urldelete");+ final boolean simulate = post.containsKey("simulate-urldelete");// parse the inputurldelete = urldelete.trim();if (urldelete_mm_subpath_checked) {// collect using url stubs- Set<String> ids = new HashSet<String>();- String[] stubURLs = urldelete.indexOf('\n') > 0 || urldelete.indexOf('\r') > 0 ? urldelete.split("[\\r\\n]+") : urldelete.split(Pattern.quote("|"));+ final Set<String> ids = new HashSet<String>();+ final String[] stubURLs = urldelete.indexOf('\n') > 0 || urldelete.indexOf('\r') > 0 ? urldelete.split("[\\r\\n]+") : urldelete.split(Pattern.quote("|"));for (String urlStub: stubURLs) {if (urlStub == null || urlStub.length() == 0) continue;- int pos = urlStub.indexOf("://",0);+ final int pos = urlStub.indexOf("://",0);if (pos == -1) {if (urlStub.startsWith("ftp")) urlStub = "ftp://" + urlStub; else urlStub = "http://" + urlStub;}try {- DigestURL u = new DigestURL(urlStub);- BlockingQueue<SolrDocument> dq = defaultConnector.concurrentDocumentsByQuery(CollectionSchema.host_s.getSolrFieldName() + ":\"" + u.getHost() + "\"", null, 0, 100000000, Long.MAX_VALUE, 100, 1, false, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());+ final DigestURL u = new DigestURL(urlStub);+ final BlockingQueue<SolrDocument> dq = defaultConnector.concurrentDocumentsByQuery(CollectionSchema.host_s.getSolrFieldName() + ":\"" + u.getHost() + "\"", null, 0, 100000000, Long.MAX_VALUE, 100, 1, false, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());SolrDocument doc;try {while ((doc = dq.take()) != AbstractSolrConnector.POISON_DOCUMENT) {- String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());+ final String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());if (url.startsWith(urlStub)) ids.add((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()));}} catch (final InterruptedException e) {@@ -161,13 +163,13 @@ public class IndexDeletion_p {} else {sb.remove(ids);defaultConnector.commit(false);- ids.forEach(urlhash -> {try {sb.index.loadTimeIndex().remove(urlhash.getBytes());} catch (IOException e) {}});+ ids.forEach(urlhash -> {try {sb.index.loadTimeIndex().remove(urlhash.getBytes());} catch (final IOException e) {}});sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs matching with " + urldelete);prop.put("urldelete-active", 2);}} else {// collect using a regular expression on urls- String regexquery = CollectionSchema.sku.getSolrFieldName() + ":/" + urldelete + "/";+ final String regexquery = CollectionSchema.sku.getSolrFieldName() + ":/" + urldelete + "/";if (simulate) {try {count = (int) defaultConnector.getCountByQuery("{!cache=false}" + regexquery);@@ -178,7 +180,7 @@ public class IndexDeletion_p {try {defaultConnector.deleteByQuery(regexquery);defaultConnector.commit(false);- try {sb.index.loadTimeIndex().clear();} catch (IOException e) {}+ try {sb.index.loadTimeIndex().clear();} catch (final IOException e) {}sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, regex match = " + urldelete);} catch (final IOException e) {}@@ -192,9 +194,9 @@ public class IndexDeletion_p {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);- boolean simulate = post.containsKey("simulate-timedelete");+ final boolean simulate = post.containsKey("simulate-timedelete");Date deleteageDate = null;- long t = timeParser(timedelete_number, timedelete_unit); // year, month, day, hour+ final long t = timeParser(timedelete_number, timedelete_unit); // year, month, day, hourif (t > 0) deleteageDate = new Date(t);final String collection1Query = (timedelete_source_loaddate_checked ? CollectionSchema.load_date_dt : CollectionSchema.last_modified).getSolrFieldName() + ":[* TO " + ISO8601Formatter.FORMATTER.format(deleteageDate) + "]";final String webgraphQuery = (timedelete_source_loaddate_checked ? WebgraphSchema.load_date_dt : WebgraphSchema.last_modified).getSolrFieldName() + ":[* TO " + ISO8601Formatter.FORMATTER.format(deleteageDate) + "]";@@ -208,7 +210,7 @@ public class IndexDeletion_p {try {defaultConnector.deleteByQuery(collection1Query);defaultConnector.commit(false);- try {sb.index.loadTimeIndex().clear();} catch (IOException e) {}+ try {sb.index.loadTimeIndex().clear();} catch (final IOException e) {}if (webgraphConnector != null) webgraphConnector.deleteByQuery(webgraphQuery);sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs older than " + timedelete_number + " " + timedelete_unit);} catch (final IOException e) {@@ -222,9 +224,9 @@ public class IndexDeletion_p {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);- boolean simulate = post.containsKey("simulate-collectiondelete");+ final boolean simulate = post.containsKey("simulate-collectiondelete");collectiondelete = collectiondelete.replaceAll(" ","").replaceAll(",", "|");- String query = collectiondelete_mode_unassigned_checked ? "-" + CollectionSchema.collection_sxt + AbstractSolrConnector.CATCHALL_DTERM : collectiondelete.length() == 0 ? CollectionSchema.collection_sxt + ":\"\"" : QueryModifier.parseCollectionExpression(collectiondelete);+ final String query = collectiondelete_mode_unassigned_checked ? "-" + CollectionSchema.collection_sxt + AbstractSolrConnector.CATCHALL_DTERM : collectiondelete.length() == 0 ? CollectionSchema.collection_sxt + ":\"\"" : QueryModifier.parseCollectionExpression(collectiondelete);if (simulate) {try {count = (int) defaultConnector.getCountByQuery(query);@@ -235,7 +237,7 @@ public class IndexDeletion_p {try {defaultConnector.deleteByQuery(query);defaultConnector.commit(false);- try {sb.index.loadTimeIndex().clear();} catch (IOException e) {}+ try {sb.index.loadTimeIndex().clear();} catch (final IOException e) {}sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, collection " + collectiondelete);} catch (final IOException e) {}@@ -248,9 +250,9 @@ public class IndexDeletion_p {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);- boolean simulate = post.containsKey("simulate-querydelete");+ final boolean simulate = post.containsKey("simulate-querydelete");- SolrConnector connector = schemaName.equals(CollectionSchema.CORE_NAME) ? defaultConnector : sb.index.fulltext().getWebgraphConnector();+ final SolrConnector connector = schemaName.equals(CollectionSchema.CORE_NAME) ? defaultConnector : sb.index.fulltext().getWebgraphConnector();if (simulate) {try {count = (int) connector.getCountByQuery(querydelete);@@ -262,7 +264,7 @@ public class IndexDeletion_p {ConcurrentLog.info("IndexDeletion", "delete by query \"" + querydelete + "\", size before deletion = " + connector.getSize());connector.deleteByQuery(querydelete);connector.commit(false);- try {sb.index.loadTimeIndex().clear();} catch (IOException e) {}+ try {sb.index.loadTimeIndex().clear();} catch (final IOException e) {}ConcurrentLog.info("IndexDeletion", "delete by query \"" + querydelete + "\", size after commit = " + connector.getSize());sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, solr query, q = " + querydelete);} catch (final IOException e) {diff --git a/htroot/IndexExport_p.java b/source/net/yacy/htroot/IndexExport_p.javasimilarity index 97%rename from htroot/IndexExport_p.javarename to source/net/yacy/htroot/IndexExport_p.javaindex 2ef08ea6c..78cc94132 100644--- a/htroot/IndexExport_p.java+++ b/source/net/yacy/htroot/IndexExport_p.java@@ -21,6 +21,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.util.List;@@ -46,13 +48,13 @@ public class IndexExport_p {final serverObjects prop = new serverObjects();- Segment segment = sb.index;+ final Segment segment = sb.index;// we have two counts of document: total number and such that are exportable with status code 200- long ucount = segment.fulltext().collectionSize();+ final long ucount = segment.fulltext().collectionSize();long ucount200 = ucount;try {ucount200 = segment.fulltext().getDefaultConnector().getCountByQuery(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200");- } catch (IOException e1) {}+ } catch (final IOException e1) {}// set default valuesprop.put("otherHosts", "");@@ -129,7 +131,7 @@ public class IndexExport_p {// start the exporttry {export = sb.index.fulltext().export(format, filter, query, maxseconds, new File(path), dom, text);- } catch (IOException e) {+ } catch (final IOException e) {prop.put("lurlexporterror", 1);prop.put("lurlexporterror_exportfile", "-no export-");prop.put("lurlexporterror_exportfailmsg", e.getMessage());diff --git a/htroot/IndexFederated_p.java b/source/net/yacy/htroot/IndexFederated_p.javasimilarity index 90%rename from htroot/IndexFederated_p.javarename to source/net/yacy/htroot/IndexFederated_p.javaindex 557d5d33b..fbe9de77b 100644--- a/htroot/IndexFederated_p.java+++ b/source/net/yacy/htroot/IndexFederated_p.java@@ -18,6 +18,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.io.BufferedReader;import java.io.ByteArrayInputStream;import java.io.IOException;@@ -47,13 +49,13 @@ public class IndexFederated_p {// return variable that accumulates replacementsfinal serverObjects prop = new serverObjects();final Switchboard sb = (Switchboard) env;-+if (post != null && post.containsKey("setrwi")) {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);-+//yacy- boolean post_core_rwi = post.getBoolean("core.service.rwi");+ final boolean post_core_rwi = post.getBoolean("core.service.rwi");final boolean previous_core_rwi = sb.index.connectedRWI() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false);env.setConfig(SwitchboardConstants.CORE_SERVICE_RWI, post_core_rwi);if (previous_core_rwi && !post_core_rwi) sb.index.disconnectRWI(); // switch off@@ -67,8 +69,8 @@ public class IndexFederated_p {if (post != null && post.containsKey("setcitation")) {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);-- boolean post_core_citation = post.getBoolean(SwitchboardConstants.CORE_SERVICE_CITATION);++ final boolean post_core_citation = post.getBoolean(SwitchboardConstants.CORE_SERVICE_CITATION);final boolean previous_core_citation = sb.index.connectedCitation() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false);env.setConfig(SwitchboardConstants.CORE_SERVICE_CITATION, post_core_citation);if (previous_core_citation && !post_core_citation) sb.index.disconnectCitation(); // switch off@@ -77,16 +79,16 @@ public class IndexFederated_p {final long fileSizeMax = (OS.isWindows) ? sb.getConfigLong("filesize.max.win", Integer.MAX_VALUE) : sb.getConfigLong( "filesize.max.other", Integer.MAX_VALUE);sb.index.connectCitation(wordCacheMaxCount, fileSizeMax);} catch (final IOException e) { ConcurrentLog.logException(e); } // switch on- boolean webgraph = post.getBoolean(SwitchboardConstants.CORE_SERVICE_WEBGRAPH);+ final boolean webgraph = post.getBoolean(SwitchboardConstants.CORE_SERVICE_WEBGRAPH);sb.index.fulltext().setUseWebgraph(webgraph);env.setConfig(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, webgraph);}-+if (post != null && post.containsKey("setsolr")) {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);-- boolean post_core_fulltext = post.getBoolean(SwitchboardConstants.CORE_SERVICE_FULLTEXT);++ final boolean post_core_fulltext = post.getBoolean(SwitchboardConstants.CORE_SERVICE_FULLTEXT);final boolean previous_core_fulltext = sb.index.fulltext().connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, SwitchboardConstants.CORE_SERVICE_FULLTEXT_DEFAULT);env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext);@@ -99,7 +101,7 @@ public class IndexFederated_p {// switch ontry { sb.index.fulltext().connectLocalSolr(); } catch (final IOException e) { ConcurrentLog.logException(e); }}-+// solrfinal boolean solrRemoteWasOn = sb.index.fulltext().connectedRemoteSolr()&& env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED,@@ -107,10 +109,10 @@ public class IndexFederated_p {String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote") & solrurls.length() > 0;env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards);-+env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED, post.getBoolean(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED));-+final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls))));final StringBuilder s = new StringBuilder();String s0;@@ -128,10 +130,10 @@ public class IndexFederated_p {}solrurls = s.toString().trim();env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, solrurls);- String shardMethodName = post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name()));- ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);+ final String shardMethodName = post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name()));+ final ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, shardMethod.name());-+if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) {// switch offtry {@@ -140,7 +142,7 @@ public class IndexFederated_p {ConcurrentLog.logException(e);}}-+final boolean writeEnabled = post.getBoolean("solr.indexing.solrremote.writeenabled");sb.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, writeEnabled);@@ -155,10 +157,10 @@ public class IndexFederated_p {final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);-+try {if (usesolr) {- ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);+ final ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);sb.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);} else {sb.index.fulltext().disconnectRemoteSolr();@@ -174,11 +176,11 @@ public class IndexFederated_p {} catch (final SolrException e) {ConcurrentLog.severe("IndexFederated_p", "change of solr connection failed", e);}-- boolean lazy = post.getBoolean("solr.indexing.lazy");++ final boolean lazy = post.getBoolean("solr.indexing.lazy");env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy);}-+/* Acquire a transaction token for the next POST form submission */prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));@@ -189,16 +191,16 @@ public class IndexFederated_p {prop.put("table", 1);final SolrConnector solr = sb.index.fulltext().getDefaultRemoteSolrConnector();final long[] size = new long[]{((RemoteSolrConnector) solr).getSize()};-+final boolean toExternalAddress = !header.accessFromLocalhost();String externalHost = null;if(toExternalAddress) {- /* The request does not come from the same computer than this peer : we get the external address used to reach it from the request headers, and we will use+ /* The request does not come from the same computer than this peer : we get the external address used to reach it from the request headers, and we will use* it if some remote solr instance(s) URLs are configured with loopback address alias (such as 'localhost', '127.0.0.1', '[::1]')* to render the externally reachable Solr instance(s) admin URLs */externalHost = header.getServerName();}-+final ArrayList<String> urls = ((ShardInstance) ((RemoteSolrConnector) solr).getInstance()).getAdminInterfaces(toExternalAddress, externalHost);boolean dark = false;@@ -222,10 +224,10 @@ public class IndexFederated_p {prop.put(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED + ".checked",env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT));- prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));- String thisShardingMethodName = env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());+ prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));+ final String thisShardingMethodName = env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());int mc = 0;- for (ShardSelection.Method method: ShardSelection.Method.values()) {+ for (final ShardSelection.Method method: ShardSelection.Method.values()) {prop.put("solr.indexing.sharding.methods_" + mc + "_method", method.name());prop.put("solr.indexing.sharding.methods_" + mc + "_description", method.description);prop.put("solr.indexing.sharding.methods_" + mc + "_selected", method.name().equals(thisShardingMethodName) ? 1 : 0);@@ -234,7 +236,7 @@ public class IndexFederated_p {prop.put("solr.indexing.sharding.methods", mc);prop.put("solr.indexing.solrremote.writeenabled.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true));prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0);-+// return rewrite propertiesreturn prop;}diff --git a/htroot/IndexImportMediawiki_p.java b/source/net/yacy/htroot/IndexImportMediawiki_p.javasimilarity index 93%rename from htroot/IndexImportMediawiki_p.javarename to source/net/yacy/htroot/IndexImportMediawiki_p.javaindex b45a76a5e..faf30ff00 100644--- a/htroot/IndexImportMediawiki_p.java+++ b/source/net/yacy/htroot/IndexImportMediawiki_p.java@@ -22,6 +22,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.net.MalformedURLException;@@ -96,18 +98,18 @@ public class IndexImportMediawiki_p {prop.put("import_prevReport", MediawikiImporter.job != null ? 1 : 0);if (post == null) {prop.put("import_status", 0);-+/* Acquire a transaction token for the next POST form submission */final String token = TransactionManager.getTransactionToken(header);prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, token);prop.put("import_" + TransactionManager.TRANSACTION_TOKEN_PARAM, token);-+} else {if (post.containsKey("file")) {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);-- String file = post.get("file");++ final String file = post.get("file");MultiProtocolURL sourceURL = null;int status = 0;String sourceFilePath = "";@@ -129,18 +131,18 @@ public class IndexImportMediawiki_p {status = 4;}}-+if (status == 0 && post.getBoolean("iffresh")) {- long lastModified = getLastModified(sourceURL);+ final long lastModified = getLastModified(sourceURL);if (lastExecutionDate != null && lastModified != 0L && Instant.ofEpochMilli(lastModified).isBefore(lastExecutionDate.toInstant())) {status = 5;prop.put("import_status_lastImportDate", GenericFormatter.formatSafely(lastExecutionDate.toInstant(), GenericFormatter.FORMAT_SIMPLE));-+/* the import is not performed, but we increase here the api call count */if(sb.tables != null) {- byte[] lastExecutedCallPk = lastExecutedCall.getPK();+ final byte[] lastExecutedCallPk = lastExecutedCall.getPK();if(lastExecutedCallPk != null && !post.containsKey(WorkTables.TABLE_API_COL_APICALL_PK)) {post.add(WorkTables.TABLE_API_COL_APICALL_PK, UTF8.String(lastExecutedCallPk));}@@ -148,7 +150,7 @@ public class IndexImportMediawiki_p {}}}- } catch (MalformedURLException e) {+ } catch (final MalformedURLException e) {status = 1;}if (status == 0) {@@ -156,14 +158,14 @@ public class IndexImportMediawiki_p {if(sb.tables != null) {/* We avoid creating a duplicate of any already recorded API call with the same parameters */if(lastExecutedCall != null && !post.containsKey(WorkTables.TABLE_API_COL_APICALL_PK)) {- byte[] lastExecutedCallPk = lastExecutedCall.getPK();+ final byte[] lastExecutedCallPk = lastExecutedCall.getPK();if(lastExecutedCallPk != null) {post.add(WorkTables.TABLE_API_COL_APICALL_PK, UTF8.String(lastExecutedCallPk));}}sb.tables.recordAPICall(post, "IndexImportMediawiki_p.html", WorkTables.TABLE_API_TYPE_DUMP, "MediaWiki Dump Import for " + sourceURL);}-+MediawikiImporter.job = new MediawikiImporter(sourceURL, sb.surrogatesInPath);MediawikiImporter.job.start();prop.put("import_dump", MediawikiImporter.job.source());@@ -173,7 +175,7 @@ public class IndexImportMediawiki_p {} else {prop.put("import_status", status);prop.put("import_status_sourceFile", sourceFilePath);-+/* Acquire a transaction token for the next POST form submission */final String token = TransactionManager.getTransactionToken(header);prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, token);@@ -190,23 +192,23 @@ public class IndexImportMediawiki_p {}return prop;}-+/*** @param fileURL the file URL. Must not be null.* @return the last modified date for the file at fileURL, or 0L when unknown or when an error occurred*/- private static long getLastModified(MultiProtocolURL fileURL) {+ private static long getLastModified(final MultiProtocolURL fileURL) {try {if (fileURL.isHTTP() || fileURL.isHTTPS()) {/* http(s) : we do not use MultiprotocolURL.lastModified() which always returns 0L for these protocols */try (HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {- HttpResponse headResponse = httpClient.HEADResponse(fileURL, false);+ final HttpResponse headResponse = httpClient.HEADResponse(fileURL, false);if (headResponse != null && headResponse.getStatusLine() != null&& headResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {- Header lastModifiedHeader = headResponse+ final Header lastModifiedHeader = headResponse.getFirstHeader(HeaderFramework.LAST_MODIFIED);if (lastModifiedHeader != null) {- Date lastModifiedDate = HeaderFramework.parseHTTPDate(lastModifiedHeader.getValue());+ final Date lastModifiedDate = HeaderFramework.parseHTTPDate(lastModifiedHeader.getValue());if(lastModifiedDate != null) {return lastModifiedDate.getTime();}@@ -216,7 +218,7 @@ public class IndexImportMediawiki_p {} else {return fileURL.lastModified();}- } catch (IOException ignored) {+ } catch (final IOException ignored) {ConcurrentLog.warn("IndexImportMediawiki_p", "Could not retrieve last modified date for dump file at " + fileURL);}return 0l;diff --git a/htroot/IndexImportOAIPMHList_p.java b/source/net/yacy/htroot/IndexImportOAIPMHList_p.javasimilarity index 85%rename from htroot/IndexImportOAIPMHList_p.javarename to source/net/yacy/htroot/IndexImportOAIPMHList_p.javaindex 6c8859540..1aa31bebc 100644--- a/htroot/IndexImportOAIPMHList_p.java+++ b/source/net/yacy/htroot/IndexImportOAIPMHList_p.java@@ -22,6 +22,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.ArrayList;import java.util.List;import java.util.Set;@@ -37,7 +39,7 @@ import net.yacy.server.serverSwitch;public class IndexImportOAIPMHList_p {- public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, serverObjects post, final serverSwitch env) {+ public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {final serverObjects prop = new serverObjects();final Switchboard sb = (Switchboard) env;@@ -46,7 +48,7 @@ public class IndexImportOAIPMHList_p {prop.put("source", 0);if (post != null && post.containsKey("source")) {- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));final Set<String> oaiRoots = new OAIListFriendsLoader().getListFriends(sb.loader, agent).keySet();boolean dark = false;@@ -66,9 +68,9 @@ public class IndexImportOAIPMHList_p {if (post != null && post.containsKey("import")) {final List<OAIPMHImporter> jobs = new ArrayList<OAIPMHImporter>();- for (OAIPMHImporter job: OAIPMHImporter.runningJobs.keySet()) jobs.add(job);- for (OAIPMHImporter job: OAIPMHImporter.startedJobs.keySet()) jobs.add(job);- for (OAIPMHImporter job: OAIPMHImporter.finishedJobs.keySet()) jobs.add(job);+ for (final OAIPMHImporter job: OAIPMHImporter.runningJobs.keySet()) jobs.add(job);+ for (final OAIPMHImporter job: OAIPMHImporter.startedJobs.keySet()) jobs.add(job);+ for (final OAIPMHImporter job: OAIPMHImporter.finishedJobs.keySet()) jobs.add(job);boolean dark = false;int count = 0;diff --git a/htroot/IndexImportOAIPMH_p.java b/source/net/yacy/htroot/IndexImportOAIPMH_p.javasimilarity index 91%rename from htroot/IndexImportOAIPMH_p.javarename to source/net/yacy/htroot/IndexImportOAIPMH_p.javaindex 6db80a0c5..df9f8a92c 100644--- a/htroot/IndexImportOAIPMH_p.java+++ b/source/net/yacy/htroot/IndexImportOAIPMH_p.java@@ -22,6 +22,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.IOException;import java.net.MalformedURLException;import java.util.ArrayList;@@ -62,7 +64,7 @@ public class IndexImportOAIPMH_p {DigestURL url = null;try {url = new DigestURL(oaipmhurl);- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));final OAIPMHLoader r = new OAIPMHLoader(sb.loader, url, sb.surrogatesInPath, agent);final ResumptionToken rt = r.getResumptionToken();prop.put("import-one", 1);@@ -97,7 +99,7 @@ public class IndexImportOAIPMH_p {DigestURL url = null;try {url = new DigestURL(oaipmhurl);- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));final OAIPMHImporter job = new OAIPMHImporter(sb.loader, agent, url);job.start();prop.put("status", 1);@@ -130,7 +132,7 @@ public class IndexImportOAIPMH_p {// start jobs for the sourcesDigestURL url = null;- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));while (!sourceList.isEmpty()) {final String oaipmhurl = sourceList.remove(r.nextInt(sourceList.size()));try {diff --git a/htroot/IndexImportWarc_p.java b/source/net/yacy/htroot/IndexImportWarc_p.javasimilarity index 87%rename from htroot/IndexImportWarc_p.javarename to source/net/yacy/htroot/IndexImportWarc_p.javaindex d5564bbe0..325454605 100644--- a/htroot/IndexImportWarc_p.java+++ b/source/net/yacy/htroot/IndexImportWarc_p.java@@ -16,6 +16,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.net.MalformedURLException;@@ -49,15 +51,15 @@ public class IndexImportWarc_p {prop.put("import", 0);if (post != null) {if (post.containsKey("file") || post.containsKey("url")) {- String filename = post.get("file");+ final String filename = post.get("file");if (filename != null && filename.length() > 0) {final File sourcefile = new File(filename);if (sourcefile.exists()) {try {- WarcImporter wi = new WarcImporter(sourcefile);+ final WarcImporter wi = new WarcImporter(sourcefile);wi.start();prop.put("import_thread", "started");- } catch (IOException ex) {+ } catch (final IOException ex) {prop.put("import_thread", "Error: file not found [" + filename + "]");}prop.put("import", 1);@@ -67,16 +69,16 @@ public class IndexImportWarc_p {prop.put("import_thread", "Error: file not found [" + filename + "]");}} else {- String urlstr = post.get("url");+ final String urlstr = post.get("url");if (urlstr != null && urlstr.length() > 0) {try {- MultiProtocolURL url = new MultiProtocolURL(urlstr);- WarcImporter wi = new WarcImporter(url);+ final MultiProtocolURL url = new MultiProtocolURL(urlstr);+ final WarcImporter wi = new WarcImporter(url);wi.start();prop.put("import_thread", "started");- } catch (MalformedURLException ex) {+ } catch (final MalformedURLException ex) {prop.put("import_thread", ex.getMessage());- } catch (IOException ex) {+ } catch (final IOException ex) {prop.put("import_thread", ex.getMessage());}prop.put("import", 1);diff --git a/htroot/IndexReIndexMonitor_p.java b/source/net/yacy/htroot/IndexReIndexMonitor_p.javasimilarity index 97%rename from htroot/IndexReIndexMonitor_p.javarename to source/net/yacy/htroot/IndexReIndexMonitor_p.javaindex 80ed6f0a6..78e40aac1 100644--- a/htroot/IndexReIndexMonitor_p.java+++ b/source/net/yacy/htroot/IndexReIndexMonitor_p.java@@ -17,6 +17,8 @@* along with this program in the file lgpl21.txt If not, see* <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.io.IOException;import java.io.UnsupportedEncodingException;import java.net.URLEncoder;@@ -44,7 +46,7 @@ import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;public class IndexReIndexMonitor_p {-+/** This servlet name, used for identifying recorded API calls */private static final String SERVLET_NAME = IndexReIndexMonitor_p.class.getSimpleName() + ".html";@@ -52,7 +54,7 @@ public class IndexReIndexMonitor_p {final Switchboard sb = (Switchboard) env;final serverObjects prop = new serverObjects();-+/* Acquire a transaction token for the next possible POST form submissions */final String nextTransactionToken = TransactionManager.getTransactionToken(header);prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken);@@ -64,15 +66,15 @@ public class IndexReIndexMonitor_p {if (post != null && post.containsKey("reindexnow") && sb.index.fulltext().connectedLocalSolr()) {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);-+migration.reindexToschema(sb);prop.put("querysize", "0");prop.put("infomessage","reindex job started");-+reidxbt = sb.getThread(ReindexSolrBusyThread.THREAD_NAME); //get new created job for following posts- }+ }}-+if (reidxbt != null) {prop.put("reindexjobrunning", 1);prop.put("querysize", reidxbt.getJobCount());@@ -94,17 +96,17 @@ public class IndexReIndexMonitor_p {prop.put("reindexjobrunning_fieldlist", 0);}}-+if (post != null && post.containsKey("stopreindex")) {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);-+sb.terminateThread(ReindexSolrBusyThread.THREAD_NAME, false);prop.put("infomessage", "reindex job stopped");prop.put("reindexjobrunning",0);} else {prop.put("infomessage", "reindex is running");- }+ }} else {prop.put("reindexjobrunning", 0);if (sb.index.fulltext().connectedLocalSolr()) {@@ -115,29 +117,29 @@ public class IndexReIndexMonitor_p {prop.putHTML("infomessage", "! reindex works only with embedded Solr index !");}}-+processReindexReport(header, sb, prop, reidxbt instanceof ReindexSolrBusyThread ? (ReindexSolrBusyThread)reidxbt : null);// recrawl job handlingBusyThread recrawlbt = sb.getThread(RecrawlBusyThread.THREAD_NAME);-+String recrawlQuery = RecrawlBusyThread.DEFAULT_QUERY;boolean inclerrdoc = RecrawlBusyThread.DEFAULT_INCLUDE_FAILED;boolean deleteOnRecrawl = RecrawlBusyThread.DEFAULT_DELETE_ON_RECRAWL;// to signal that a setting shall change the form provides a fixed parameter setup=recrawljob, if not present return status onlyif (post != null && "recrawljob".equals(post.get("setup"))) { // it's a command to recrawlThread-+/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);-+if(post.containsKey("recrawlquerytext")) {recrawlQuery = post.get("recrawlquerytext");}-+if (post.containsKey("includefailedurls")) {inclerrdoc = post.getBoolean("includefailedurls");}-+if (post.containsKey("deleteOnRecrawl")) {deleteOnRecrawl = post.getBoolean("deleteOnRecrawl");}@@ -149,14 +151,14 @@ public class IndexReIndexMonitor_p {sb.deployThread(RecrawlBusyThread.THREAD_NAME, "ReCrawl", "recrawl existing documents", null,new RecrawlBusyThread(Switchboard.getSwitchboard(), recrawlQuery, inclerrdoc, deleteOnRecrawl), 1000);recrawlbt = sb.getThread(RecrawlBusyThread.THREAD_NAME);-+/* store this call as an api call for easy scheduling possibility */if(sb.tables != null) {/* We avoid creating a duplicate of any already recorded API call with the same parameters */final Row lastExecutedCall = WorkTables.selectLastExecutedApiCall(IndexReIndexMonitor_p.SERVLET_NAME, post, sb);if (lastExecutedCall != null && !post.containsKey(WorkTables.TABLE_API_COL_APICALL_PK)) {- byte[] lastExecutedCallPk = lastExecutedCall.getPK();+ final byte[] lastExecutedCallPk = lastExecutedCall.getPK();if (lastExecutedCallPk != null) {post.add(WorkTables.TABLE_API_COL_APICALL_PK, UTF8.String(lastExecutedCallPk));}@@ -193,7 +195,7 @@ public class IndexReIndexMonitor_p {prop.put("recrawljobrunning_simulationResult", 3);}}-+if(post.containsKey("recrawlDefaults")) {recrawlQuery = RecrawlBusyThread.DEFAULT_QUERY;inclerrdoc = RecrawlBusyThread.DEFAULT_INCLUDE_FAILED;@@ -217,9 +219,9 @@ public class IndexReIndexMonitor_p {}}}-+processRecrawlReport(header, sb, prop, (RecrawlBusyThread)recrawlbt);-+// just post status of recrawlThreadif (recrawlbt != null && !recrawlbt.shutdownInProgress()) { // provide statusprop.put("recrawljobrunning", 1);@@ -237,11 +239,11 @@ public class IndexReIndexMonitor_p {// return rewrite propertiesreturn prop;}-+/*** Write information on the eventual currently running or last reindex job* terminated- *+ ** @param header* current request header. Must not be null.* @param sb@@ -294,10 +296,10 @@ public class IndexReIndexMonitor_p {final serverObjects prop, final RecrawlBusyThread recrawlbt) {if (recrawlbt != null) {prop.put("recrawlReport", 1);-+prop.put("recrawlReport_error", recrawlbt.isTerminatedBySolrFailure());--++int jobStatus;if(recrawlbt.isAlive()) {if(recrawlbt.shutdownInProgress()) {@@ -309,12 +311,12 @@ public class IndexReIndexMonitor_p {jobStatus = 2; // Terminated}prop.put("recrawlReport_jobStatus", jobStatus);-+prop.put("recrawlReport_recrawlquerytext", recrawlbt.getQuery());-+Locale formatLocale;if (sb != null) {- String lng = sb.getConfig("locale.language", Locale.ENGLISH.getLanguage());+ final String lng = sb.getConfig("locale.language", Locale.ENGLISH.getLanguage());if ("browser".equals(lng)) {/* Only use the client locale when locale.language is set to browser */formatLocale = header.getLocale();diff --git a/htroot/IndexSchema_p.java b/source/net/yacy/htroot/IndexSchema_p.javasimilarity index 84%rename from htroot/IndexSchema_p.javarename to source/net/yacy/htroot/IndexSchema_p.javaindex c40b1e8c0..ae00ede54 100644--- a/htroot/IndexSchema_p.java+++ b/source/net/yacy/htroot/IndexSchema_p.java@@ -18,6 +18,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.util.Iterator;@@ -40,11 +42,11 @@ public class IndexSchema_p {// return variable that accumulates replacementsfinal serverObjects prop = new serverObjects();final Switchboard sb = (Switchboard) env;-+String schemaName = CollectionSchema.CORE_NAME;- if (post != null) schemaName = post.get("core", schemaName);- SchemaConfiguration cs = schemaName.equals(CollectionSchema.CORE_NAME) ? sb.index.fulltext().getDefaultConfiguration() : sb.index.fulltext().getWebgraphConfiguration();-+ if (post != null) schemaName = post.get("core", schemaName);+ final SchemaConfiguration cs = schemaName.equals(CollectionSchema.CORE_NAME) ? sb.index.fulltext().getDefaultConfiguration() : sb.index.fulltext().getWebgraphConfiguration();+if (post != null && post.containsKey("set")) {// read index schema table flagsfinal Iterator<SchemaConfiguration.Entry> i = cs.entryIterator();@@ -69,7 +71,7 @@ public class IndexSchema_p {entry.setEnable(c);modified = true;}- }+ }}if (modified) { // save settings to config file if modifiedtry {@@ -78,11 +80,11 @@ public class IndexSchema_p {sb.index.fulltext().getWebgraphConfiguration().commit();modified = false;} catch (final IOException ex) {}- }+ }}-+if (post != null && post.containsKey("resetselectiontodefault")) {- // reset Solr field selection to default configuration+ // reset Solr field selection to default configurationFile solrInitFile;if (cs instanceof WebgraphConfiguration) { // get default configuration for webgraphsolrInitFile = new File(sb.getAppPath(), "defaults/" + Switchboard.SOLR_WEBGRAPH_CONFIGURATION_NAME);@@ -90,10 +92,10 @@ public class IndexSchema_p {solrInitFile = new File(sb.getAppPath(), "defaults/" + Switchboard.SOLR_COLLECTION_CONFIGURATION_NAME);}try {- SchemaConfiguration solrConfigurationInit = new SchemaConfiguration(solrInitFile);- Iterator<SchemaConfiguration.Entry> it = cs.entryIterator(); // get current configuration+ final SchemaConfiguration solrConfigurationInit = new SchemaConfiguration(solrInitFile);+ final Iterator<SchemaConfiguration.Entry> it = cs.entryIterator(); // get current configurationwhile (it.hasNext()) { // iterate over entries and enable/disable according to default- SchemaConfiguration.Entry etr = it.next();+ final SchemaConfiguration.Entry etr = it.next();etr.setEnable(solrConfigurationInit.contains(etr.key()));}cs.commit();@@ -101,26 +103,26 @@ public class IndexSchema_p {ConcurrentLog.warn("IndexSchema", "file " + solrInitFile.getAbsolutePath() + " not found");}}-+/* Acquire a transaction token for the next possible POST IndexReIndeMonitor_p form submission */prop.put("IndexReIndex_" + TransactionManager.TRANSACTION_TOKEN_PARAM,TransactionManager.getTransactionToken(header, "/IndexReIndexMonitor_p.html"));-+int c = 0;boolean dark = false;// use enum SolrField to keep defined order- SchemaDeclaration[] cc = schemaName.equals(CollectionSchema.CORE_NAME) ? CollectionSchema.values() : WebgraphSchema.values();+ final SchemaDeclaration[] cc = schemaName.equals(CollectionSchema.CORE_NAME) ? CollectionSchema.values() : WebgraphSchema.values();String filterstr = null;// set active filter button property- boolean viewall = (post == null) || (filterstr = post.get("filter")) == null;- boolean viewactiveonly = !viewall && "active".equals(filterstr);- boolean viewdisabledonly = !viewall && "disabled".equals(filterstr);+ final boolean viewall = (post == null) || (filterstr = post.get("filter")) == null;+ final boolean viewactiveonly = !viewall && "active".equals(filterstr);+ final boolean viewdisabledonly = !viewall && "disabled".equals(filterstr);prop.put("viewall",viewall);prop.put("activeonly", viewactiveonly);prop.put("disabledonly", viewdisabledonly);-- for(SchemaDeclaration field : cc) {- boolean showline = viewactiveonly ? cs.contains(field.name()) : (viewdisabledonly ? cs.containsDisabled(field.name()): true);++ for(final SchemaDeclaration field : cc) {+ final boolean showline = viewactiveonly ? cs.contains(field.name()) : (viewdisabledonly ? cs.containsDisabled(field.name()): true);if (showline) {prop.put("schema_" + c + "_dark", dark ? 1 : 0); dark = !dark;prop.put("schema_" + c + "_checked", cs.contains(field.name()) ? 1 : 0);@@ -139,7 +141,7 @@ public class IndexSchema_p {prop.put("cores_" + 1 + "_selected", WebgraphSchema.CORE_NAME.equals(schemaName) ? 1 : 0);prop.put("cores", 2);prop.put("core", schemaName);-+// return rewrite propertiesreturn prop;}diff --git a/htroot/IndexShare_p.java b/source/net/yacy/htroot/IndexShare_p.javasimilarity index 95%rename from htroot/IndexShare_p.javarename to source/net/yacy/htroot/IndexShare_p.javaindex 6f17a2749..10f722ec2 100644--- a/htroot/IndexShare_p.java+++ b/source/net/yacy/htroot/IndexShare_p.java@@ -28,6 +28,8 @@//import java.net.*;//import java.io.*;//import de.anomic.tools.*;+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.search.Switchboard;import net.yacy.search.SwitchboardConstants;@@ -43,7 +45,7 @@ public class IndexShare_p {final serverObjects prop = new serverObjects();// get segment- Segment indexSegment = sb.index;+ final Segment indexSegment = sb.index;if (post == null) {prop.put("linkfreq", sb.getConfigLong("defaultLinkReceiveFrequency",30));diff --git a/htroot/Load_MediawikiWiki.java b/source/net/yacy/htroot/Load_MediawikiWiki.javasimilarity index 98%rename from htroot/Load_MediawikiWiki.javarename to source/net/yacy/htroot/Load_MediawikiWiki.javaindex 8af1c8fe1..62f0bc994 100644--- a/htroot/Load_MediawikiWiki.java+++ b/source/net/yacy/htroot/Load_MediawikiWiki.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Set;import net.yacy.cora.protocol.RequestHeader;@@ -49,7 +51,7 @@ public class Load_MediawikiWiki {}prop.put("starturl", "http://");prop.put("address", myUrl);-+// hidden form param : clean up search events cache ?if (post != null && post.containsKey("cleanSearchCache")) {prop.put("cleanSearchCacheChecked", post.getBoolean("cleanSearchCache"));diff --git a/htroot/Load_PHPBB3.java b/source/net/yacy/htroot/Load_PHPBB3.javasimilarity index 99%rename from htroot/Load_PHPBB3.javarename to source/net/yacy/htroot/Load_PHPBB3.javaindex 899394ae1..3291380d6 100644--- a/htroot/Load_PHPBB3.java+++ b/source/net/yacy/htroot/Load_PHPBB3.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Set;import net.yacy.cora.protocol.RequestHeader;@@ -50,7 +52,7 @@ public class Load_PHPBB3 {final boolean intranet = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "").equals("intranet");prop.put("starturl", (intranet) ? myUrl : "http://");prop.put("address", myUrl);-+// hidden form param : clean up search events cache ?if (post != null && post.containsKey("cleanSearchCache")) {prop.put("cleanSearchCacheChecked", post.getBoolean("cleanSearchCache"));diff --git a/htroot/Load_RSS_p.java b/source/net/yacy/htroot/Load_RSS_p.javasimilarity index 95%rename from htroot/Load_RSS_p.javarename to source/net/yacy/htroot/Load_RSS_p.javaindex 531368380..05975966c 100644--- a/htroot/Load_RSS_p.java+++ b/source/net/yacy/htroot/Load_RSS_p.java@@ -18,6 +18,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.io.IOException;import java.net.MalformedURLException;import java.text.DateFormat;@@ -72,8 +74,8 @@ public class Load_RSS_p {final Switchboard sb = (Switchboard)env;final String collection = post == null ? "user" : CommonPattern.SPACE.matcher(post.get("collection", "user").trim()).replaceAll("");- Map<String, Pattern> collections = CrawlProfile.collectionParser(collection);- boolean collectionEnabled = sb.index.fulltext().getDefaultConfiguration().isEmpty() || sb.index.fulltext().getDefaultConfiguration().contains(CollectionSchema.collection_sxt);+ final Map<String, Pattern> collections = CrawlProfile.collectionParser(collection);+ final boolean collectionEnabled = sb.index.fulltext().getDefaultConfiguration().isEmpty() || sb.index.fulltext().getDefaultConfiguration().contains(CollectionSchema.collection_sxt);prop.put("showload_collectionEnabled", collectionEnabled ? 1 : 0);prop.put("showload_collection", collection);prop.put("showload", 0);@@ -165,7 +167,7 @@ public class Load_RSS_p {}if (post != null && post.containsKey("addSelectedFeedScheduler")) {- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));for (final Map.Entry<String, String> entry: post.entrySet()) {if (entry.getValue().startsWith(CHECKBOX_ITEM_PREFIX)) {Row row;@@ -274,7 +276,7 @@ public class Load_RSS_p {ConcurrentLog.warn("Load_RSS", "url not well-formed: '" + post.get("url", "") + "'");}- ClientIdentification.Agent agent = post == null ? ClientIdentification.yacyInternetCrawlerAgent : ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));+ final ClientIdentification.Agent agent = post == null ? ClientIdentification.yacyInternetCrawlerAgent : ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));// if we have an url then try to load the rssRSSReader rss = null;@@ -305,7 +307,7 @@ public class Load_RSS_p {DigestURL messageUrl;try {messageUrl = new DigestURL(message.getLink());- } catch (MalformedURLException e) {+ } catch (final MalformedURLException e) {ConcurrentLog.warn("Load_RSS", "Malformed feed item link URL : " + message.getLink());continue loop;}@@ -323,7 +325,7 @@ public class Load_RSS_p {DigestURL mediaUrl;try {mediaUrl = new DigestURL(message.getEnclosure());- } catch (MalformedURLException e) {+ } catch (final MalformedURLException e) {ConcurrentLog.warn("Load_RSS", "Malformed feed item enclosure URL : " + message.getEnclosure());continue loop;}@@ -337,7 +339,7 @@ public class Load_RSS_p {final List<DigestURL> urlsToIndex = new ArrayList<DigestURL>();loop: for (final Map.Entry<String, DigestURL> entry: hash2UrlMap.entrySet()) {final DigestURL messageUrl = entry.getValue();- HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(messageUrl.hash()));+ final HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(messageUrl.hash()));if (harvestProcess != null) {continue loop;}@@ -410,7 +412,7 @@ public class Load_RSS_p {pubDate = item.getPubDate();if(link != null && StringUtils.isNotEmpty(item.getGuid())) {- HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(link.hash()));+ final HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(link.hash()));prop.put("showitems_item_" + i + "_hasLink", true);prop.putHTML("showitems_item_" + i + "_hasLink_link", link.toNormalform(true));@@ -433,7 +435,7 @@ public class Load_RSS_p {i++;if(enclosure != null && enclosure != link && StringUtils.isNotEmpty(item.getGuid())) {- HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(enclosure.hash()));+ final HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(enclosure.hash()));prop.put("showitems_item_" + i + "_hasLink", true);prop.putHTML("showitems_item_" + i + "_hasLink_link", enclosure.toNormalform(true));diff --git a/htroot/MessageSend_p.java b/source/net/yacy/htroot/MessageSend_p.javasimilarity index 95%rename from htroot/MessageSend_p.javarename to source/net/yacy/htroot/MessageSend_p.javaindex 57271bc00..9569f778f 100644--- a/htroot/MessageSend_p.java+++ b/source/net/yacy/htroot/MessageSend_p.java@@ -25,6 +25,8 @@//javac -classpath .:../Classes MessageSend_p.java//if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.IOException;import java.net.MalformedURLException;import java.util.Date;@@ -78,7 +80,7 @@ public class MessageSend_p {// open an editor page for the message// first ask if the other peer is online, and also what kind of document it accepts- Seed seed = sb.peers.getConnected(ASCII.getBytes(hash));+ final Seed seed = sb.peers.getConnected(ASCII.getBytes(hash));if (seed != null) {for (final String ip : seed.getIPs()) {Map<String, String> result = null;@@ -132,7 +134,7 @@ public class MessageSend_p {sb.peers.peerActions.interfaceDeparture(targetPeer, ip);}} else {-+prop.put("mode_permission", "1");// write input form@@ -172,7 +174,7 @@ public class MessageSend_p {if (subject.length() > 100) subject = subject.substring(0, 100);if (message.length() > messagesize) message = message.substring(0, messagesize);final byte[] mb = UTF8.getBytes(message);- SeedDB seedDB = sb.peers;+ final SeedDB seedDB = sb.peers;// prepare requestfinal String salt = crypt.randomSalt();@@ -183,7 +185,7 @@ public class MessageSend_p {parts.put("subject", UTF8.StringBody(subject));parts.put("message", UTF8.StringBody(mb));final Seed seed = seedDB.getConnected(ASCII.getBytes(hash));- boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,+ final boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);Post post1 = null;for(final String ip : seed.getIPs()) {@@ -209,7 +211,7 @@ public class MessageSend_p {}}}-+if (post1 != null) {break;}diff --git a/htroot/Messages_p.java b/source/net/yacy/htroot/Messages_p.javasimilarity index 97%rename from htroot/Messages_p.javarename to source/net/yacy/htroot/Messages_p.javaindex 40540d281..26d5f513b 100644--- a/htroot/Messages_p.java+++ b/source/net/yacy/htroot/Messages_p.java@@ -24,12 +24,16 @@// javac -classpath .:../Classes Message.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.util.Date;import java.util.Iterator;import java.util.TreeMap;+import com.google.common.io.Files;+import net.yacy.cora.date.GenericFormatter;import net.yacy.cora.document.encoding.UTF8;import net.yacy.cora.protocol.HeaderFramework;@@ -42,8 +46,6 @@ import net.yacy.search.SwitchboardConstants;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;-import com.google.common.io.Files;-public class Messages_p {diff --git a/htroot/Network.java b/source/net/yacy/htroot/Network.javasimilarity index 95%rename from htroot/Network.javarename to source/net/yacy/htroot/Network.javaindex 5e993fc68..64519f686 100644--- a/htroot/Network.java+++ b/source/net/yacy/htroot/Network.java@@ -27,6 +27,8 @@// javac -classpath .:../classes Network.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.net.MalformedURLException;import java.util.HashMap;import java.util.HashSet;@@ -68,11 +70,11 @@ public class Network {final long start = System.currentTimeMillis();final servletProperties prop = new servletProperties();-+prop.put("menu", post == null ? 2 : (post.get("menu", "").equals("embed")) ? 0 : (post.get("menu","").equals("simple")) ? 1 : 2);if (sb.peers.mySeed() != null) prop.put("menu_newpeer_peerhash", sb.peers.mySeed().hash);- String pathInfo = requestHeader.getPathInfo();+ final String pathInfo = requestHeader.getPathInfo();prop.setLocalized(!pathInfo.endsWith(".xml") && !pathInfo.endsWith(".json"));prop.putHTML("page_networkTitle", sb.getConfig("network.unit.description", "unspecified"));prop.putHTML("page_networkName", sb.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"));@@ -128,10 +130,10 @@ public class Network {accActLinks += LCount;accActWords += ICount;}- String port = seed.get(Seed.PORT, "-");- Set<String> ips = seed.getIPs();+ final String port = seed.get(Seed.PORT, "-");+ final Set<String> ips = seed.getIPs();int ipsc = 0;- for (String s: ips) {+ for (final String s: ips) {prop.put("table_ips_" + ipsc + "_nodestate", seed.getFlagRootNode() ? 1 : 0);prop.put("table_ips_" + ipsc + "_c", 0);prop.putHTML("table_ips_" + ipsc + "_c_hash", seed.hash);@@ -221,27 +223,27 @@ public class Network {}final ConcurrentMap<String, String> map = new ConcurrentHashMap<String, String>();- String challengeIP = post.get("peerIP");- String challengePort = post.get("peerPort");+ final String challengeIP = post.get("peerIP");+ final String challengePort = post.get("peerPort");map.put(Seed.IP, challengeIP);map.put(Seed.PORT, challengePort);Seed peer = post.get("peerHash") == null ? null : new Seed(post.get("peerHash"), map);sb.updateMySeed();- Seed mySeed = sb.peers.mySeed();+ final Seed mySeed = sb.peers.mySeed();final String challengeURLStr = peer.getPublicURL(challengeIP, false);try {final MultiProtocolURL challengeURL = new MultiProtocolURL(challengeURLStr);final Map<String, String> response = Protocol.hello(mySeed, sb.peers.peerActions, challengeURL, peer.hash);-+if (response == null) {- Seed peerd = sb.peers.get(peer.hash);+ final Seed peerd = sb.peers.get(peer.hash);if (peerd != null) peer = peerd;sb.peers.peerActions.interfaceDeparture(peer, challengeIP);prop.put("table_comment",1);prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from <a href=\"" + challengeURL + "\" target=\"_blank\">" + challengeURL + "</a>");} else {- String yourtype = response.get("yourtype");- String yourip = response.get("yourip");+ final String yourtype = response.get("yourtype");+ final String yourip = response.get("yourip");peer = sb.peers.getConnected(peer.hash);if (peer == null) {prop.put("table_comment",1);@@ -436,11 +438,11 @@ public class Network {}if (location.length() > 10) location = location.substring(0, 10);if (location.length() == 0) {- Locale l = Domains.getLocale(ip);+ final Locale l = Domains.getLocale(ip);if (l != null) location = l.toString();}prop.putHTML(STR_TABLE_LIST + conCount + "_location", location);- String port = seed.get(Seed.PORT, "-");+ final String port = seed.get(Seed.PORT, "-");int ipsc = 0;for (final String s: ips) {prop.put(STR_TABLE_LIST + conCount + "_ips_" + ipsc + "_nodestate", seed.getFlagRootNode() ? 1 : 0);@@ -506,7 +508,7 @@ public class Network {final String peertags = MapTools.set2string(seed.getPeerTags(), ",", false);prop.putHTML(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", ((peertags == null) || (peertags.isEmpty())) ? "no tags given" : ("tags = " + peertags));}- String[] yv = yacyVersion.combined2prettyVersion(seed.get(Seed.VERSION, "0.1"), shortname);+ final String[] yv = yacyVersion.combined2prettyVersion(seed.get(Seed.VERSION, "0.1"), shortname);prop.putHTML(STR_TABLE_LIST + conCount + "_version", yv[0] + "/" + yv[1]);prop.putNum(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen);prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(Seed.UTC, "-"));@@ -550,9 +552,9 @@ public class Network {if(path != null && path.endsWith(".xml")) {final ResponseHeader outgoingHeader = new ResponseHeader(200);outgoingHeader.put(HeaderFramework.CORS_ALLOW_ORIGIN, "*");- prop.setOutgoingHeader(outgoingHeader);+ prop.setOutgoingHeader(outgoingHeader);}-+// return rewrite propertiesreturn prop;}diff --git a/htroot/NetworkHistory.java b/source/net/yacy/htroot/NetworkHistory.javasimilarity index 85%rename from htroot/NetworkHistory.javarename to source/net/yacy/htroot/NetworkHistory.javaindex da7833f97..043ef3b7f 100644--- a/htroot/NetworkHistory.java+++ b/source/net/yacy/htroot/NetworkHistory.java@@ -18,6 +18,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.io.IOException;import java.text.ParseException;import java.util.ArrayList;@@ -42,7 +44,7 @@ import net.yacy.visualization.RasterPlotter;public class NetworkHistory {-+public static RasterPlotter respond(@SuppressWarnings("unused") final RequestHeader header, serverObjects post, final serverSwitch env) {final Switchboard sb = (Switchboard) env;@@ -63,22 +65,22 @@ public class NetworkHistory {cI size of the index (number of documents)*/final Set<String> columns = new LinkedHashSet<>();- for (String col: columnsx) columns.add(col);+ for (final String col: columnsx) columns.add(col);// scan the database and put in values- List<Map<String, Long>> rows = new ArrayList<>(maxtime * 2);- long now = System.currentTimeMillis();- long timelimit = now - maxtime * 3600000L;+ final List<Map<String, Long>> rows = new ArrayList<>(maxtime * 2);+ final long now = System.currentTimeMillis();+ final long timelimit = now - maxtime * 3600000L;try {- Iterator<Row> rowi = sb.tables.iterator("stats", false);+ final Iterator<Row> rowi = sb.tables.iterator("stats", false);Map <String, Long> statrow;while (rowi.hasNext()) {- Row row = rowi.next();- String d = ASCII.String(row.getPK());- Date date = GenericFormatter.SHORT_MINUTE_FORMATTER.parse(d, 0).getTime();+ final Row row = rowi.next();+ final String d = ASCII.String(row.getPK());+ final Date date = GenericFormatter.SHORT_MINUTE_FORMATTER.parse(d, 0).getTime();if (date.getTime() < timelimit) break;statrow = new HashMap<>();- for (String key: columns) {- byte[] x = row.get(key);+ for (final String key: columns) {+ final byte[] x = row.get(key);if (x != null) statrow.put(key, Long.parseLong(ASCII.String(x)));}statrow.put("time", date.getTime());@@ -87,12 +89,12 @@ public class NetworkHistory {} catch (final IOException|ParseException e) {ConcurrentLog.logException(e);}-+// find correct scaleint maxpeers = 100, minpeers = Integer.MAX_VALUE; // to be measured by pre-scanning the db- for (Map<String, Long> row: rows) {- for (String column: columns) {- Long v = row.get(column);+ for (final Map<String, Long> row: rows) {+ for (final String column: columns) {+ final Long v = row.get(column);if (v != null) maxpeers = Math.max(maxpeers, (int) v.longValue());if (v != null && v.longValue() > 0) minpeers = Math.min(minpeers, (int) v.longValue());}@@ -132,24 +134,24 @@ public class NetworkHistory {if (columns.contains("cP")) headline += ", POTENTIAL JUNIOR PEERS";if (columns.contains("cI")) headline = "YACY PEER '" + sb.peers.myName().toUpperCase() + "' INDEX SIZE HISTORY: NUMBER OF DOCUMENTS";if (columns.contains("cR")) headline = "YACY PEER '" + sb.peers.myName().toUpperCase() + "' INDEX SIZE HISTORY: NUMBER OF RWI ENTRIES";- ChartPlotter chart = new ChartPlotter(width, height, 0xFFFFFFl, 0x000000l, 0xAAAAAAl, leftborder, rightborder, topborder, bottomborder, headline, "IN THE LAST " + timestr);+ final ChartPlotter chart = new ChartPlotter(width, height, 0xFFFFFFl, 0x000000l, 0xAAAAAAl, leftborder, rightborder, topborder, bottomborder, headline, "IN THE LAST " + timestr);long pps = (long)hspace * (long)bottomscale / maxtime;int pixelperscale = Math.max(8, (int)pps );chart.declareDimension(ChartPlotter.DIMENSION_BOTTOM, bottomscale, pixelperscale, -maxtime, 0x000000l, 0xCCCCCCl, "TIME/HOURS");pps = (long)vspace * (long)leftscale / (maxpeers-minpeers);pixelperscale = Math.max(8, (int)pps );chart.declareDimension(ChartPlotter.DIMENSION_LEFT, leftscale, pixelperscale, minpeers, 0x008800l, null , columns.contains("cI") ? "DOCUMENTS" : columns.contains("cR") ? "RWIs" : "PEERS");-+// write the datafloat x0, x1;int y0, y1;Long time;- for (String column: columns) {+ for (final String column: columns) {x0 = 1.0f; y0 = 0;- for (Map<String, Long> row: rows) {+ for (final Map<String, Long> row: rows) {time = row.get("time");if (time == null) continue;- Long v = row.get(column);+ final Long v = row.get(column);if (v == null) continue;x1 = (time - now) / 3600000.0f;y1 = (int) v.longValue();diff --git a/htroot/NetworkPicture.java b/source/net/yacy/htroot/NetworkPicture.javasimilarity index 96%rename from htroot/NetworkPicture.javarename to source/net/yacy/htroot/NetworkPicture.javaindex 51b8b628d..52b177f43 100644--- a/htroot/NetworkPicture.java+++ b/source/net/yacy/htroot/NetworkPicture.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.concurrent.Semaphore;import net.yacy.cora.protocol.RequestHeader;@@ -41,7 +43,7 @@ public class NetworkPicture {private static final ConcurrentLog log = new ConcurrentLog("NetworkPicture");private static final Semaphore sync = new Semaphore(1, true);private static long lastAccessSeconds = 0;-+public static EncodedImage respond(final RequestHeader header,final serverObjects post,diff --git a/htroot/News.java b/source/net/yacy/htroot/News.javasimilarity index 96%rename from htroot/News.javarename to source/net/yacy/htroot/News.javaindex 92a288958..296e75cb0 100644--- a/htroot/News.java+++ b/source/net/yacy/htroot/News.java@@ -24,10 +24,12 @@// javac -classpath .:../classes Network.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.Iterator;import java.util.Map;-import java.util.Set;import java.util.Map.Entry;+import java.util.Set;import net.yacy.cora.date.GenericFormatter;import net.yacy.cora.protocol.HeaderFramework;@@ -162,9 +164,9 @@ public class News {if(seed == null) {link = "";} else {- Set<String> ips = seed.getIPs();+ final Set<String> ips = seed.getIPs();if(!ips.isEmpty()) {- link = "http://" + seed.getPublicAddress(ips.iterator().next()) + "/Wiki.html?page=" + record.attribute("page", "");+ link = "http://" + seed.getPublicAddress(ips.iterator().next()) + "/Wiki.html?page=" + record.attribute("page", "");} else {link = "";}@@ -175,7 +177,7 @@ public class News {if(seed == null) {link = "";} else {- Set<String> ips = seed.getIPs();+ final Set<String> ips = seed.getIPs();if(!ips.isEmpty()) {link = "http://" + seed.getPublicAddress(ips.iterator().next()) + "/Blog.html?page=" + record.attribute("page", "");} else {diff --git a/htroot/PeerLoadPicture.java b/source/net/yacy/htroot/PeerLoadPicture.javasimilarity index 96%rename from htroot/PeerLoadPicture.javarename to source/net/yacy/htroot/PeerLoadPicture.javaindex 3482279cf..2f014da47 100644--- a/htroot/PeerLoadPicture.java+++ b/source/net/yacy/htroot/PeerLoadPicture.java@@ -1,3 +1,5 @@+package net.yacy.htroot;+import java.awt.Color;import java.awt.Image;import java.util.HashMap;diff --git a/htroot/PerformanceConcurrency_p.java b/source/net/yacy/htroot/PerformanceConcurrency_p.javasimilarity index 99%rename from htroot/PerformanceConcurrency_p.javarename to source/net/yacy/htroot/PerformanceConcurrency_p.javaindex 051a207e5..05cf21b8d 100644--- a/htroot/PerformanceConcurrency_p.java+++ b/source/net/yacy/htroot/PerformanceConcurrency_p.java@@ -19,6 +19,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Iterator;import net.yacy.cora.protocol.RequestHeader;diff --git a/htroot/PerformanceGraph.java b/source/net/yacy/htroot/PerformanceGraph.javasimilarity index 94%rename from htroot/PerformanceGraph.javarename to source/net/yacy/htroot/PerformanceGraph.javaindex bb1c45b9a..dc8cc7224 100644--- a/htroot/PerformanceGraph.java+++ b/source/net/yacy/htroot/PerformanceGraph.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.concurrent.TimeUnit;import net.yacy.cora.protocol.RequestHeader;@@ -37,7 +39,7 @@ public class PerformanceGraph {private static long indeSizeCache = 0;private static long indexSizeTime = 0;-+public static RasterPlotter respond(@SuppressWarnings("unused") final RequestHeader header, serverObjects post, final serverSwitch env) {final Switchboard sb = (Switchboard) env;@@ -48,12 +50,12 @@ public class PerformanceGraph {final boolean showMemory = !post.containsKey("nomem");final boolean showPeers = !post.containsKey("nopeers");- long t = System.currentTimeMillis();+ final long t = System.currentTimeMillis();if (t - indexSizeTime > 10000) {indeSizeCache = sb.index.fulltext().collectionSize();indexSizeTime = t;}- RasterPlotter graph = ProfilingGraph.performanceGraph(+ final RasterPlotter graph = ProfilingGraph.performanceGraph(width, height,indeSizeCache + " URLS / " + sb.index.RWICount() + " WORDS IN INDEX / " + sb.index.RWIBufferCount() + " WORDS IN CACHE",600, TimeUnit.SECONDS,diff --git a/htroot/PerformanceMemory_p.java b/source/net/yacy/htroot/PerformanceMemory_p.javasimilarity index 96%rename from htroot/PerformanceMemory_p.javarename to source/net/yacy/htroot/PerformanceMemory_p.javaindex 37710a4a3..4cae9ea15 100644--- a/htroot/PerformanceMemory_p.java+++ b/source/net/yacy/htroot/PerformanceMemory_p.java@@ -25,6 +25,8 @@//if the shell's current path is HTROOT//import java.util.Iterator;+package net.yacy.htroot;+import java.util.ConcurrentModificationException;import java.util.Iterator;import java.util.Map;@@ -59,7 +61,7 @@ public class PerformanceMemory_p {prop.put("gc", "1");prop.put("autoreload.checked", "1");} else if (post.containsKey("dummy")) {- boolean simulatedshortmemory = post.containsKey("simulatedshortmemory");+ final boolean simulatedshortmemory = post.containsKey("simulatedshortmemory");MemoryControl.setSimulatedShortStatus(simulatedshortmemory);if (simulatedshortmemory) prop.put("autoreload.checked", "1");final boolean std = post.containsKey("useStandardmemoryStrategy");@@ -98,7 +100,7 @@ public class PerformanceMemory_p {prop.putNum("memoryUsedNow", MemoryControl.used() / MB);// write table for Table index sizes- Iterator<String> i = Table.filenames();+ final Iterator<String> i = Table.filenames();String filename;TableStatistics stats;int p;@@ -198,7 +200,7 @@ public class PerformanceMemory_p {// prop.putNum("objectCacheStartShrink", Cache.getMemStartShrink() / (1024d * 1024d));// prop.putNum("objectHitCacheTotalMem", totalhitmem / (1024d * 1024d));// prop.putNum("objectMissCacheTotalMem", totalmissmem / (1024d * 1024d));-+prop.putNum("namecacheHit.size", Domains.nameCacheHitSize());prop.putNum("namecacheHit.Hit", Domains.cacheHit_Hit);prop.putNum("namecacheHit.Miss", Domains.cacheHit_Miss);diff --git a/htroot/PerformanceQueues_p.java b/source/net/yacy/htroot/PerformanceQueues_p.javasimilarity index 99%rename from htroot/PerformanceQueues_p.javarename to source/net/yacy/htroot/PerformanceQueues_p.javaindex b728216a2..6b274edce 100644--- a/htroot/PerformanceQueues_p.java+++ b/source/net/yacy/htroot/PerformanceQueues_p.java@@ -24,6 +24,8 @@//javac -classpath .:../classes Network.java//if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import java.util.Iterator;import java.util.Map;diff --git a/htroot/PerformanceSearch_p.java b/source/net/yacy/htroot/PerformanceSearch_p.javasimilarity index 96%rename from htroot/PerformanceSearch_p.javarename to source/net/yacy/htroot/PerformanceSearch_p.javaindex 35230934f..001fe00d7 100644--- a/htroot/PerformanceSearch_p.java+++ b/source/net/yacy/htroot/PerformanceSearch_p.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Iterator;import net.yacy.cora.protocol.RequestHeader;diff --git a/htroot/Performance_p.java b/source/net/yacy/htroot/Performance_p.javasimilarity index 95%rename from htroot/Performance_p.javarename to source/net/yacy/htroot/Performance_p.javaindex 91f5d1ea7..cb220f0ff 100644--- a/htroot/Performance_p.java+++ b/source/net/yacy/htroot/Performance_p.java@@ -24,12 +24,14 @@//javac -classpath .:../classes Network.java//if the shell's current path is HTROOT+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;public class Performance_p {-+public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch sb) {return PerformanceQueues_p.respond(header, post, sb);}diff --git a/htroot/ProxyIndexingMonitor_p.java b/source/net/yacy/htroot/ProxyIndexingMonitor_p.javasimilarity index 97%rename from htroot/ProxyIndexingMonitor_p.javarename to source/net/yacy/htroot/ProxyIndexingMonitor_p.javaindex 75704192b..92a35432d 100644--- a/htroot/ProxyIndexingMonitor_p.java+++ b/source/net/yacy/htroot/ProxyIndexingMonitor_p.java@@ -27,6 +27,8 @@// javac -classpath .:../classes ProxyIndexingMonitor_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import net.yacy.cora.protocol.RequestHeader;diff --git a/htroot/QuickCrawlLink_p.java b/source/net/yacy/htroot/QuickCrawlLink_p.javasimilarity index 96%rename from htroot/QuickCrawlLink_p.javarename to source/net/yacy/htroot/QuickCrawlLink_p.javaindex e7c666541..adf0497d1 100644--- a/htroot/QuickCrawlLink_p.java+++ b/source/net/yacy/htroot/QuickCrawlLink_p.java@@ -29,6 +29,8 @@//if the shell's current path is HTROOT+package net.yacy.htroot;+import java.net.MalformedURLException;import java.util.Date;@@ -70,7 +72,7 @@ public class QuickCrawlLink_p {} else {port = sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090);}-+prop.put("mode_host", Domains.LOCALHOST);prop.put("mode_port", port);@@ -88,7 +90,7 @@ public class QuickCrawlLink_p {crawlingStart = UTF8.decodeURL(crawlingStart);// get segment- Segment indexSegment = sb.index;+ final Segment indexSegment = sb.index;// get the browser titleString title = post.get("title", null);@@ -130,8 +132,8 @@ public class QuickCrawlLink_p {final byte[] urlhash = crawlingStartURL.hash();indexSegment.fulltext().remove(urlhash);sb.crawlQueues.noticeURL.removeByURLHash(urlhash);- int timezoneOffset = post.getInt("timezoneOffset", 0);-+ final int timezoneOffset = post.getInt("timezoneOffset", 0);+// create crawling profileCrawlProfile pe = null;try {diff --git a/htroot/RankingRWI_p.java b/source/net/yacy/htroot/RankingRWI_p.javasimilarity index 99%rename from htroot/RankingRWI_p.javarename to source/net/yacy/htroot/RankingRWI_p.javaindex 5fba47dd6..9d674e389 100644--- a/htroot/RankingRWI_p.java+++ b/source/net/yacy/htroot/RankingRWI_p.java@@ -25,6 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.LinkedHashMap;import java.util.Map;import java.util.Map.Entry;@@ -73,7 +75,7 @@ public class RankingRWI_p {rankingParameters.put(RankingProfile.WORDDISTANCE, "Word Distance;a higher ranking level prefers documents where the search words appear close together. This ranking parameter works like a NEAR operator in more-than-one word searches.");rankingParameters.put(RankingProfile.WORDSINTEXT, "Words In Text;a higher ranking level prefers documents with a large number of words. Be aware that this is a compensation of the term frequency parameter.");rankingParameters.put(RankingProfile.WORDSINTITLE, "Words In Title;a higher ranking level prefers documents with a large number of words in the document title.");-+rankingParameters.put(RankingProfile.URLCOMPINTOPLIST, "URL Component Appears In Toplist;a higher ranking level prefers documents with words in the url path that match words in the toplist. The toplist is generated dynamically from the search results using a statistic of the most used words. The toplist is a top-10 list of the most used words in URLs and document titles.");rankingParameters.put(RankingProfile.DESCRCOMPINTOPLIST, "Description Comp. Appears In Toplist;a higher ranking level prefers documents with words in the document description that match words in the toplist. The toplist is generated dynamically from the search results using a statistic of the most used words. The toplist is a top-10 list of the most used words in URLs and document titles.");rankingParameters.put(RankingProfile.PREFER, "Application Of Prefer Pattern;a higher ranking level prefers documents where the url matches the prefer pattern given in a search request.");diff --git a/htroot/RankingSolr_p.java b/source/net/yacy/htroot/RankingSolr_p.javasimilarity index 77%rename from htroot/RankingSolr_p.javarename to source/net/yacy/htroot/RankingSolr_p.javaindex 9905ff2e6..f94480e11 100644--- a/htroot/RankingSolr_p.java+++ b/source/net/yacy/htroot/RankingSolr_p.java@@ -18,14 +18,19 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.io.IOException;import java.util.ArrayList;import java.util.Collection;import java.util.HashMap;import java.util.LinkedHashMap;import java.util.Map;-import java.util.TreeMap;import java.util.Map.Entry;+import java.util.TreeMap;++import org.apache.solr.common.params.CommonParams;+import org.apache.solr.common.params.DisMaxParams;import net.yacy.cora.federate.solr.Ranking;import net.yacy.cora.federate.solr.SchemaDeclaration;@@ -41,9 +46,6 @@ import net.yacy.search.schema.CollectionSchema;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;-import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.params.DisMaxParams;-public class RankingSolr_p {@@ -56,18 +58,18 @@ public class RankingSolr_p {int profileNr = 0;if (post != null) profileNr = post.getInt("profileNr", profileNr);-+if (post != null && post.containsKey("EnterBoosts")) {- StringBuilder boostString = new StringBuilder(); // SwitchboardConstants.SEARCH_RANKING_SOLR_BOOST;- for (Map.Entry<String, String> entry: post.entrySet()) {+ final StringBuilder boostString = new StringBuilder(); // SwitchboardConstants.SEARCH_RANKING_SOLR_BOOST;+ for (final Map.Entry<String, String> entry: post.entrySet()) {if (entry.getKey().startsWith("boost")) {- String fieldName = entry.getKey().substring(6);- CollectionSchema field = CollectionSchema.valueOf(fieldName);+ final String fieldName = entry.getKey().substring(6);+ final CollectionSchema field = CollectionSchema.valueOf(fieldName);if (field == null) continue;- String fieldValue = entry.getValue();+ final String fieldValue = entry.getValue();if (fieldValue == null || fieldValue.length() == 0) continue;try {- float boost = Float.parseFloat(fieldValue);+ final float boost = Float.parseFloat(fieldValue);if (boost > 0.0f) { // don't allow <= 0if (boostString.length() > 0) boostString.append(',');boostString.append(field.getSolrFieldName()).append('^').append(Float.toString(boost));@@ -78,26 +80,26 @@ public class RankingSolr_p {}}if (boostString.length() > 0) {- String s = boostString.toString();+ final String s = boostString.toString();sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTFIELDS_ + profileNr, s);sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).updateBoosts(s);}}if (post != null && post.containsKey("ResetBoosts")) {- String s = "url_paths_sxt^3.0,synonyms_sxt^0.5,title^5.0,text_t^1.0,host_s^6.0,h1_txt^5.0,url_file_name_tokens_t^4.0,h2_txt^3.0,keywords^2.0,description_txt^1.5,author^1.0";+ final String s = "url_paths_sxt^3.0,synonyms_sxt^0.5,title^5.0,text_t^1.0,host_s^6.0,h1_txt^5.0,url_file_name_tokens_t^4.0,h2_txt^3.0,keywords^2.0,description_txt^1.5,author^1.0";sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTFIELDS_ + profileNr, s);sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).updateBoosts(s);}if (post != null && post.containsKey("EnterBQ")) {- String bq = post.get(DisMaxParams.BQ);+ final String bq = post.get(DisMaxParams.BQ);if (bq != null) {sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTQUERY_ + profileNr, bq);sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setBoostQuery(bq);}}if (post != null && post.containsKey("ResetBQ")) {- String bq = "crawldepth_i:0^0.8\ncrawldepth_i:1^0.4";+ final String bq = "crawldepth_i:0^0.8\ncrawldepth_i:1^0.4";if (bq != null) {sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTQUERY_ + profileNr, bq);sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setBoostQuery(bq);@@ -105,14 +107,14 @@ public class RankingSolr_p {}if (post != null && post.containsKey("EnterFQ")) {- String fq = post.get(CommonParams.FQ);+ final String fq = post.get(CommonParams.FQ);if (fq != null) {sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_FILTERQUERY_ + profileNr, fq);sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setFilterQuery(fq);}}if (post != null && post.containsKey("ResetFQ")) {- String fq = ""; // i.e. "http_unique_b:true AND www_unique_b:true"+ final String fq = ""; // i.e. "http_unique_b:true AND www_unique_b:true"if (fq != null) {sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_FILTERQUERY_ + profileNr, fq);sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setFilterQuery(fq);@@ -120,14 +122,14 @@ public class RankingSolr_p {}if (post != null && post.containsKey("EnterBF")) {- String bf = post.get(DisMaxParams.BF);+ final String bf = post.get(DisMaxParams.BF);if (bf != null) {sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTFUNCTION_ + profileNr, bf);sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setBoostFunction(bf);}}if (post != null && post.containsKey("ResetBF")) {- String bf = "";+ final String bf = "";if (bf != null) {sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTFUNCTION_ + profileNr, bf);sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setBoostFunction(bf);@@ -137,11 +139,11 @@ public class RankingSolr_p {final serverObjects prop = new serverObjects();int i = 0;- CollectionConfiguration colcfg = sb.index.fulltext().getDefaultConfiguration();- Ranking ranking = colcfg.getRanking(profileNr);- for (SchemaDeclaration field: CollectionSchema.values()) {+ final CollectionConfiguration colcfg = sb.index.fulltext().getDefaultConfiguration();+ final Ranking ranking = colcfg.getRanking(profileNr);+ for (final SchemaDeclaration field: CollectionSchema.values()) {if (!field.isSearchable()) continue;- Float boost = ranking.getFieldBoost(field);+ final Float boost = ranking.getFieldBoost(field);if (boost != null || colcfg.contains(field)) { // show only available or configured boost fieldsprop.put("boosts_" + i + "_field", field.getSolrFieldName());if (boost == null || boost.floatValue() <= 0.0f) {@@ -169,12 +171,12 @@ public class RankingSolr_p {}prop.put("profiles", 4);prop.put("profileNr", profileNr);-+// make boost hints for vocabulariesMap<String, ReversibleScoreMap<String>> vocabularyFacet;try {vocabularyFacet = sb.index.fulltext().getDefaultConnector().getFacets(CollectionSchema.vocabularies_sxt.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM, 100, CollectionSchema.vocabularies_sxt.getSolrFieldName());- } catch (IOException e) {+ } catch (final IOException e) {ConcurrentLog.logException(e);vocabularyFacet = new HashMap<>();}@@ -183,14 +185,14 @@ public class RankingSolr_p {} else {prop.put("boosthint", 1);prop.putHTML("boosthint_vocabulariesfield", CollectionSchema.vocabularies_sxt.getSolrFieldName());- ReversibleScoreMap<String> vokcounts = vocabularyFacet.values().iterator().next();- Collection<String> vocnames = vokcounts.keyList(true);+ final ReversibleScoreMap<String> vokcounts = vocabularyFacet.values().iterator().next();+ final Collection<String> vocnames = vokcounts.keyList(true);prop.putHTML("boosthint_vocabulariesavailable", vocnames.toString());- ArrayList<String> voccountFields = new ArrayList<>();- ArrayList<String> voclogcountFields = new ArrayList<>();- ArrayList<String> voclogcountsFields = new ArrayList<>();- ArrayList<String> ff = new ArrayList<>();- for (String vocname: vocnames) {+ final ArrayList<String> voccountFields = new ArrayList<>();+ final ArrayList<String> voclogcountFields = new ArrayList<>();+ final ArrayList<String> voclogcountsFields = new ArrayList<>();+ final ArrayList<String> ff = new ArrayList<>();+ for (final String vocname: vocnames) {voccountFields.add(CollectionSchema.VOCABULARY_PREFIX + vocname + CollectionSchema.VOCABULARY_COUNT_SUFFIX);voclogcountFields.add(CollectionSchema.VOCABULARY_PREFIX + vocname + CollectionSchema.VOCABULARY_LOGCOUNT_SUFFIX);voclogcountsFields.add(CollectionSchema.VOCABULARY_PREFIX + vocname + CollectionSchema.VOCABULARY_LOGCOUNTS_SUFFIX);@@ -200,18 +202,18 @@ public class RankingSolr_p {prop.putHTML("boosthint_vocabulariesvoccount", voccountFields.toString());prop.putHTML("boosthint_vocabulariesvoclogcount", voclogcountFields.toString());prop.putHTML("boosthint_vocabulariesvoclogcounts", voclogcountsFields.toString());- String[] facetfields = ff.toArray(new String[ff.size()]);+ final String[] facetfields = ff.toArray(new String[ff.size()]);int fc = 0;if (facetfields.length > 0) try {- LinkedHashMap<String, ReversibleScoreMap<String>> facets = sb.index.fulltext().getDefaultConnector().getFacets("*:*", 100, facetfields);+ final LinkedHashMap<String, ReversibleScoreMap<String>> facets = sb.index.fulltext().getDefaultConnector().getFacets("*:*", 100, facetfields);facets.put(CollectionSchema.vocabularies_sxt.getSolrFieldName(), vokcounts);- for (Map.Entry<String, ReversibleScoreMap<String>> facetentry: facets.entrySet()) {- ReversibleScoreMap<String> facetfieldmap = facetentry.getValue();+ for (final Map.Entry<String, ReversibleScoreMap<String>> facetentry: facets.entrySet()) {+ final ReversibleScoreMap<String> facetfieldmap = facetentry.getValue();if (facetfieldmap.size() == 0) continue;- TreeMap<String, Integer> statMap = new TreeMap<>();- for (String k: facetfieldmap) statMap.put(k, facetfieldmap.get(k));+ final TreeMap<String, Integer> statMap = new TreeMap<>();+ for (final String k: facetfieldmap) statMap.put(k, facetfieldmap.get(k));prop.put("boosthint_facets_" + fc + "_facetname", facetentry.getKey());- int c = 0; for (Entry<String, Integer> entry: statMap.entrySet()) {+ int c = 0; for (final Entry<String, Integer> entry: statMap.entrySet()) {prop.put("boosthint_facets_" + fc + "_facet_" + c + "_key", entry.getKey());prop.put("boosthint_facets_" + fc + "_facet_" + c + "_count", entry.getValue());c++;@@ -219,11 +221,11 @@ public class RankingSolr_p {prop.put("boosthint_facets_" + fc + "_facet", c);fc++;}- } catch (IOException e) {+ } catch (final IOException e) {}prop.put("boosthint_facets", fc);}-+return prop;}diff --git a/htroot/RegexTest.java b/source/net/yacy/htroot/RegexTest.javasimilarity index 90%rename from htroot/RegexTest.javarename to source/net/yacy/htroot/RegexTest.javaindex 6902e93af..c7a563337 100644--- a/htroot/RegexTest.java+++ b/source/net/yacy/htroot/RegexTest.java@@ -7,17 +7,19 @@* modify it under the terms of the GNU Lesser General Public* License as published by the Free Software Foundation; either* version 2.1 of the License, or (at your option) any later version.- *+ ** This library is distributed in the hope that it will be useful,* but WITHOUT ANY WARRANTY; without even the implied warranty of* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU* Lesser General Public License for more details.- *+ ** You should have received a copy of the GNU Lesser General Public License* along with this program in the file lgpl21.txt* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.util.regex.PatternSyntaxException;import net.yacy.cora.protocol.RequestHeader;@@ -25,11 +27,11 @@ import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;public class RegexTest {-+public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {- String text = (post == null) ? "" : post.get("text", "");- String regex = (post == null) ? ".*" : post.get("regex", ".*");+ final String text = (post == null) ? "" : post.get("text", "");+ final String regex = (post == null) ? ".*" : post.get("regex", ".*");String error = "";Boolean match = null;try {@@ -37,9 +39,9 @@ public class RegexTest {} catch (final PatternSyntaxException e) {error = e.getMessage();}-+final serverObjects prop = new serverObjects();-+prop.put("text", text);prop.put("regex", regex);prop.put("match", match == null ? 2 : (match.booleanValue() ? 1 : 0));diff --git a/htroot/RemoteCrawl_p.java b/source/net/yacy/htroot/RemoteCrawl_p.javasimilarity index 92%rename from htroot/RemoteCrawl_p.javarename to source/net/yacy/htroot/RemoteCrawl_p.javaindex 1117d47cf..102e71ad0 100644--- a/htroot/RemoteCrawl_p.java+++ b/source/net/yacy/htroot/RemoteCrawl_p.java@@ -26,6 +26,8 @@// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Iterator;import java.util.regex.Matcher;import java.util.regex.Pattern;@@ -53,7 +55,7 @@ public class RemoteCrawl_p {sb.tables.recordAPICall(post, "RemoteCrawl_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "remote crawler configuration");if (post.containsKey("crawlResponse")) {- boolean crawlResponse = post.get("crawlResponse", "off").equals("on");+ final boolean crawlResponse = post.get("crawlResponse", "off").equals("on");// read remote crawl request settingssb.initRemoteCrawler(crawlResponse);@@ -71,11 +73,11 @@ public class RemoteCrawl_p {// set seed information directlysb.peers.mySeed().setFlagAcceptRemoteCrawl(sb.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false));-+// write remote crawl request settingsprop.put("disabled", !sb.peers.mySeed().isActive() && !sb.peers.mySeed().getFlagAcceptRemoteCrawl() ? 1 : 0);prop.put("crawlResponse", sb.peers.mySeed().getFlagAcceptRemoteCrawl() ? 1 : 0);- long RTCbusySleep = Math.max(1, env.getConfigLong(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, 100));+ final long RTCbusySleep = Math.max(1, env.getConfigLong(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, 100));final int RTCppm = (int) (60000L / RTCbusySleep);prop.put("acceptCrawlLimit", RTCppm);@@ -90,14 +92,14 @@ public class RemoteCrawl_p {Iterator<Seed> e = null;e = sb.peers.seedsSortedConnected(false, Seed.RCOUNT);//e = sb.peers.seedsSortedConnected(false, yacySeed.LCOUNT);- Pattern peerSearchPattern = null;+ final Pattern peerSearchPattern = null;while (e.hasNext() && conCount < 300) {seed = e.next();assert seed != null;if (seed != null) {final long lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);if (lastseen > 720) continue;- long rcount = seed.getLong(Seed.RCOUNT, 0);+ final long rcount = seed.getLong(Seed.RCOUNT, 0);if (rcount == 0) continue;if ((post != null && post.containsKey("search")) && peerSearchPattern != null /*(wrongregex == null)*/) {boolean abort = true;@@ -118,7 +120,7 @@ public class RemoteCrawl_p {prop.putHTML(STR_TABLE_LIST + conCount + "_shortname", shortname);prop.putHTML(STR_TABLE_LIST + conCount + "_peeraddress", peeradr);prop.put(STR_TABLE_LIST + conCount + "_age", seed.getAge());- String[] yv = yacyVersion.combined2prettyVersion(seed.get(Seed.VERSION, "0.1"), shortname);+ final String[] yv = yacyVersion.combined2prettyVersion(seed.get(Seed.VERSION, "0.1"), shortname);prop.putHTML(STR_TABLE_LIST + conCount + "_version", yv[0] + "/" + yv[1]);prop.putNum(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen);prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(Seed.UTC, "-"));diff --git a/htroot/SearchAccessRate_p.java b/source/net/yacy/htroot/SearchAccessRate_p.javasimilarity index 99%rename from htroot/SearchAccessRate_p.javarename to source/net/yacy/htroot/SearchAccessRate_p.javaindex 62ad0853b..d3b60fc87 100644--- a/htroot/SearchAccessRate_p.java+++ b/source/net/yacy/htroot/SearchAccessRate_p.java@@ -20,6 +20,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Properties;import net.yacy.cora.protocol.RequestHeader;diff --git a/htroot/SearchEventPicture.java b/source/net/yacy/htroot/SearchEventPicture.javasimilarity index 96%rename from htroot/SearchEventPicture.javarename to source/net/yacy/htroot/SearchEventPicture.javaindex 835506d01..cd58a7c24 100644--- a/htroot/SearchEventPicture.java+++ b/source/net/yacy/htroot/SearchEventPicture.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.peers.graphics.NetworkGraph;import net.yacy.search.Switchboard;diff --git a/htroot/ServerScannerList.java b/source/net/yacy/htroot/ServerScannerList.javasimilarity index 97%rename from htroot/ServerScannerList.javarename to source/net/yacy/htroot/ServerScannerList.javaindex e5dd62a26..df1defe36 100644--- a/htroot/ServerScannerList.java+++ b/source/net/yacy/htroot/ServerScannerList.java@@ -18,6 +18,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.net.MalformedURLException;import java.util.ConcurrentModificationException;import java.util.Iterator;@@ -41,7 +43,7 @@ public class ServerScannerList {final serverObjects prop = new serverObjects();final Switchboard sb = (Switchboard)env;- boolean edit = post != null && post.containsKey("edit");+ final boolean edit = post != null && post.containsKey("edit");prop.put("servertable_edit", edit ? 1 : 0);prop.put("embedded", post == null ? 0 : post.containsKey("embedded") ? 1 : 0);prop.put("servertable", 0);diff --git a/htroot/SettingsAck_p.java b/source/net/yacy/htroot/SettingsAck_p.javasimilarity index 95%rename from htroot/SettingsAck_p.javarename to source/net/yacy/htroot/SettingsAck_p.javaindex 953e506ee..735372552 100644--- a/htroot/SettingsAck_p.java+++ b/source/net/yacy/htroot/SettingsAck_p.java@@ -27,6 +27,8 @@// javac -classpath .:../Classes SettingsAck_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.HashMap;import java.util.Iterator;import java.util.StringTokenizer;@@ -66,7 +68,7 @@ public class SettingsAck_p {prop.put("info", "1");//no information submittedreturn prop;}-+/* Check this is a valid transaction */TransactionManager.checkPostTransaction(header, post);@@ -102,13 +104,13 @@ public class SettingsAck_p {if (post.containsKey("proxyaccount")) {// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=ProxyAccess");-+/** display port info*/prop.put("info_port", env.getLocalPort());prop.put("info_restart", "0");-+// read and process dataString filter = (post.get("proxyfilter")).trim();final boolean useProxyAccounts = post.containsKey("use_proxyaccounts") && post.get("use_proxyaccounts").equals("on");@@ -156,21 +158,21 @@ public class SettingsAck_p {// http networkingif (post.containsKey("httpNetworking")) {-+// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=ProxyAccess");// set transparent proxy flag- boolean isTransparentProxy = post.containsKey("isTransparentProxy");+ final boolean isTransparentProxy = post.containsKey("isTransparentProxy");env.setConfig(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, isTransparentProxy);prop.put("info_isTransparentProxy", isTransparentProxy ? "on" : "off");if (isTransparentProxy) prop.put("needsRestart", isTransparentProxy);-+// set proxyAlwaysFresh flag- boolean proxyAlwaysFresh = post.containsKey("proxyAlwaysFresh");+ final boolean proxyAlwaysFresh = post.containsKey("proxyAlwaysFresh");env.setConfig("proxyAlwaysFresh", proxyAlwaysFresh);prop.put("info_proxyAlwaysFresh", proxyAlwaysFresh ? "on" : "off");-+// setting via header propertyenv.setConfig("proxy.sendViaHeader", post.containsKey("proxy.sendViaHeader"));prop.put("info_proxy.sendViaHeader", post.containsKey("proxy.sendViaHeader")? "on" : "off");@@ -178,7 +180,7 @@ public class SettingsAck_p {// setting X-Forwarded-for header propertyenv.setConfig("proxy.sendXForwardedForHeader", post.containsKey("proxy.sendXForwardedForHeader"));prop.put("info_proxy.sendXForwardedForHeader", post.containsKey("proxy.sendXForwardedForHeader")? "on" : "off");-+prop.put("info", "20");return prop;}@@ -187,9 +189,9 @@ public class SettingsAck_p {if (post.containsKey("serveraccount")) {// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=ServerAccess");-+// fileHost- String fileHost = (post.get("fileHost")).trim();+ final String fileHost = (post.get("fileHost")).trim();if (fileHost != null && !fileHost.isEmpty() && !fileHost.equals(env.getConfig("fileHost", "localpeer"))) {env.setConfig("fileHost", fileHost);}@@ -201,7 +203,7 @@ public class SettingsAck_p {} else if (staticIP.startsWith("https://")) {if (staticIP.length() > 8) { staticIP = staticIP.substring(8); } else { staticIP = ""; }}- String error = Seed.isProperIP(staticIP) ? null : "ip not proper: " + staticIP;+ final String error = Seed.isProperIP(staticIP) ? null : "ip not proper: " + staticIP;if (error == null) {serverCore.useStaticIP = true;sb.peers.mySeed().setIP(staticIP);@@ -213,18 +215,18 @@ public class SettingsAck_p {}// publicPort- String publicPort = (post.get("publicPort")).trim();+ final String publicPort = (post.get("publicPort")).trim();try {- Integer pport = Integer.parseInt(publicPort);+ final Integer pport = Integer.parseInt(publicPort);if(pport < 65535 && pport >= 0) {serverCore.usePublicPort = true;sb.peers.mySeed().setPort(pport);env.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, publicPort);}- } catch (NumberFormatException e) {+ } catch (final NumberFormatException e) {// noop}-+// server access dataString filter = (post.get("serverfilter")).trim();/*String user = (String) post.get("serveruser");@@ -274,19 +276,19 @@ public class SettingsAck_p {prop.putHTML("info_filter", filter);return prop;}-+/* Compression settings */if (post.containsKey("serverCompression")) {// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=ServerAccess");-+final boolean oldValue = env.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT);final boolean newValue = post.containsKey(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP);env.setConfig(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP, newValue);-+prop.put("needsRestart", newValue != oldValue);-+/* server compression settings saved */prop.put("info", "37");return prop;@@ -334,7 +336,7 @@ public class SettingsAck_p {if (post.containsKey("urlproxySettings")) {// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=UrlProxyAccess");-+env.setConfig("proxyURL.access", post.get("urlproxyfilter"));env.setConfig("proxyURL.rewriteURLs", post.get("urlproxydomains"));env.setConfig("proxyURL", "on".equals(post.get("urlproxyenabled")) ? true : false);@@ -347,7 +349,7 @@ public class SettingsAck_p {if (post.containsKey("seedUploadRetry")) {// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=seed");-+String error;if ((error = Network.saveSeedList(sb)) == null) {// trying to upload the seed-list file@@ -364,7 +366,7 @@ public class SettingsAck_p {if (post.containsKey("seedSettings")) {// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=seed");-+// get the currently used uploading methodfinal String oldSeedUploadMethod = env.getConfig("seedUploadMethod","none");final String newSeedUploadMethod = post.get("seedUploadMethod");@@ -381,7 +383,7 @@ public class SettingsAck_p {env.setConfig("seedUploadMethod", newSeedUploadMethod);sb.peers.mySeed().put(Seed.SEEDLISTURL, newSeedURLStr);if (seedUrlChanged) sb.peers.saveMySeed();-+// try an uploadString error;if ((error = Network.saveSeedList(sb)) == null) {@@ -398,7 +400,7 @@ public class SettingsAck_p {return prop;}}-+if (post.containsKey("seedFileSettings") || post.containsKey("seedFtpSettings") || post.containsKey("seedScpSettings")) {// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=seed");@@ -465,7 +467,7 @@ public class SettingsAck_p {if (post.containsKey("msgForwarding")) {// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=messageForwarding");-+env.setConfig("msgForwardingEnabled", post.containsKey("msgForwardingEnabled"));env.setConfig("msgForwardingCmd", post.get("msgForwardingCmd"));env.setConfig("msgForwardingTo", post.get("msgForwardingTo"));@@ -480,7 +482,7 @@ public class SettingsAck_p {// Crawler settingsif (post.containsKey("crawlerSettings")) {-+// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=crawler");@@ -568,11 +570,11 @@ public class SettingsAck_p {// server port settingsif (post.containsKey("serverports")) {- int port = post.getInt("port", 8090);+ final int port = post.getInt("port", 8090);if (port > 0) env.setConfig(SwitchboardConstants.SERVER_PORT, port);- int portssl = post.getInt("port.ssl", 8443);+ final int portssl = post.getInt("port.ssl", 8443);if (portssl > 0) env.setConfig(SwitchboardConstants.SERVER_SSLPORT, portssl);- int portshutdown = post.getInt("port.shutdown", -1);+ final int portshutdown = post.getInt("port.shutdown", -1);env.setConfig(SwitchboardConstants.SERVER_SHUTDOWNPORT, portshutdown);prop.put("info_port", port);prop.put("info_port.ssl", portssl);@@ -584,11 +586,11 @@ public class SettingsAck_p {// change https portif (post.containsKey("port.ssl")) {-+// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=ProxyAccess");-- int port = post.getInt("port.ssl", 8443);++ final int port = post.getInt("port.ssl", 8443);if (port > 0 && port != env.getConfigInt(SwitchboardConstants.SERVER_SSLPORT, 8443)) {env.setConfig(SwitchboardConstants.SERVER_SSLPORT, port);}@@ -596,80 +598,80 @@ public class SettingsAck_p {prop.put("info", "32");return prop;}-+// Debug/Analysis settingsif (post.containsKey("debugAnalysisSettings")) {// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=debug");-+boolean tickedCheckbox = post.containsKey("solrBinaryResponse");env.setConfig(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED, tickedCheckbox);-+tickedCheckbox = post.containsKey("searchTestLocalDHT");env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, tickedCheckbox);tickedCheckbox = post.containsKey("searchTestLocalSolr");env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, tickedCheckbox);-+tickedCheckbox = post.containsKey("searchShowRanking");env.setConfig(SwitchboardConstants.SEARCH_RESULT_SHOW_RANKING, tickedCheckbox);-+tickedCheckbox = post.containsKey(SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED);sb.setConfig(SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED, tickedCheckbox);TextSnippet.statistics.setEnabled(tickedCheckbox);-- /* For easier user understanding, the following flags controlling data sources selection++ /* For easier user understanding, the following flags controlling data sources selection* are rendered in the UI as checkboxes corresponding to enabled value when ticked */tickedCheckbox = post.containsKey("searchLocalDHT");env.setConfig(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_OFF, !tickedCheckbox);-+tickedCheckbox = post.containsKey("searchLocalSolr");env.setConfig(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, !tickedCheckbox);-+tickedCheckbox = post.containsKey("searchRemoteDHT");env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, !tickedCheckbox);tickedCheckbox = post.containsKey("searchRemoteSolr");env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, !tickedCheckbox);-+/* Let's clean up all search events as these settings affect how search is performed and we don't* want cached results obtained with the previous settings */SearchEventCache.cleanupEvents(true);-+prop.put("info", "34");return prop;}-+// Referrer Policy settingsif (post.containsKey("referrerPolicySettings")) {// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=referrer");-- String metaPolicy = post.get("metaPolicy", SwitchboardConstants.REFERRER_META_POLICY_DEFAULT);++ final String metaPolicy = post.get("metaPolicy", SwitchboardConstants.REFERRER_META_POLICY_DEFAULT);env.setConfig(SwitchboardConstants.REFERRER_META_POLICY, metaPolicy);-- boolean tickedCheckbox = post.containsKey("searchResultNoReferrer");++ final boolean tickedCheckbox = post.containsKey("searchResultNoReferrer");env.setConfig(SwitchboardConstants.SEARCH_RESULT_NOREFERRER, tickedCheckbox);-+prop.put("info", "35");return prop;}-+// HTTP client settingsif (post.containsKey("httpClientSettings")) {// set backlinkprop.put("needsRestart_referer", "Settings_p.html?page=httpClient");-+if(System.getProperty("jsse.enableSNIExtension") == null) {/* Only apply custom SNI extension settings when the JVM system option jsse.enableSNIExtension is not defined */env.setConfig(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED,post.containsKey(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED));-+env.setConfig(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED,post.containsKey(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED));}sb.initOutgoingConnectionSettings();-+prop.put("info", "38");return prop;}diff --git a/htroot/Settings_p.java b/source/net/yacy/htroot/Settings_p.javasimilarity index 95%rename from htroot/Settings_p.javarename to source/net/yacy/htroot/Settings_p.javaindex 329843394..fc08514e6 100644--- a/htroot/Settings_p.java+++ b/source/net/yacy/htroot/Settings_p.java@@ -22,6 +22,8 @@// javac -classpath .:../Classes Settings_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.HashMap;import java.util.Iterator;@@ -40,19 +42,19 @@ import net.yacy.server.serverSwitch;public final class Settings_p {-+public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {// return variable that accumulates replacementsfinal serverObjects prop = new serverObjects();final Switchboard sb = (Switchboard) env;-+//if (post == null) System.out.println("POST: NULL"); else System.out.println("POST: " + post.toString());-+final String page = (post == null) ? "general" : post.get("page", "general");-+/* Acquire a transaction token for the next possible POST form submission */prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header, "/SettingsAck_p.html"));-+if (page.equals("ProxyAccess")) {prop.put("settingsTables", "Settings_ProxyAccess.inc");}@@ -62,9 +64,9 @@ public final class Settings_p {else if (page.equals("UrlProxyAccess")) {prop.put("settingsTables", "Settings_UrlProxyAccess.inc");}- else if (page.equals("ServerAccess")) {+ else if (page.equals("ServerAccess")) {prop.put("settingsTables", "Settings_ServerAccess.inc");- } else if (page.equals("referrer")) {+ } else if (page.equals("referrer")) {prop.put("settingsTables", "Settings_Referrer.inc");}else if (page.equals("SystemBehaviour")) {@@ -90,7 +92,7 @@ public final class Settings_p {}prop.put("port", env.getLocalPort());-+prop.putHTML("peerName", sb.peers.mySeed().getName());prop.putHTML("staticIP", env.getConfig("staticIP", ""));prop.putHTML("publicPort", env.getConfig("publicPort",""));@@ -98,28 +100,28 @@ public final class Settings_p {String peerLang = env.getConfig("locale.language", "default");if (peerLang.equals("default")) peerLang = "en";prop.putHTML("peerLang", peerLang);-+// http networking settingsprop.put("isTransparentProxy", env.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "1" : "0");prop.put("proxyAlwaysFresh", env.getConfigBool("proxyAlwaysFresh", false) ? "1" : "0");prop.put("proxy.sendViaHeader", env.getConfigBool("proxy.sendViaHeader", false) ? "1" : "0");prop.put("proxy.sendXForwardedForHeader", env.getConfigBool("proxy.sendXForwardedForHeader", true) ? "1" : "0");-+// remote proxyprop.put("remoteProxyUseChecked", env.getConfigBool("remoteProxyUse", false) ? 1 : 0);prop.put("remoteProxyUse4SSL", env.getConfigBool("remoteProxyUse4SSL", true) ? 1 : 0);-+prop.putHTML("remoteProxyHost", env.getConfig("remoteProxyHost", ""));prop.putHTML("remoteProxyPort", env.getConfig("remoteProxyPort", ""));-+prop.putHTML("remoteProxyUser", env.getConfig("remoteProxyUser", ""));prop.putHTML("remoteProxyPwd", env.getConfig("remoteProxyPwd", ""));-+prop.putHTML("remoteProxyNoProxy", env.getConfig("remoteProxyNoProxy", ""));-+// proxy access filterprop.putHTML("proxyfilter", env.getConfig("proxyClient", "*"));-+// proxy passwordif (!env.getConfigBool("use_proxyAccounts", false)) {// no password has been specified@@ -143,47 +145,47 @@ public final class Settings_p {// server access filterprop.putHTML("serverfilter", env.getConfig("serverClient", "*"));-+/* gzip compression of HTTP responses */prop.put(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,env.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT));-+// server passwordprop.put("serveruser","server");-+// clientIPprop.putXML("clientIP", header.getRemoteAddr() == null ? "<unknown>" : header.getRemoteAddr());- /*+ /** seed upload settings*/// available methodsString enabledUploader = env.getConfig("seedUploadMethod", "none");-+// for backward compatiblity ....- if ((enabledUploader.equalsIgnoreCase("Ftp")) ||+ if ((enabledUploader.equalsIgnoreCase("Ftp")) ||((enabledUploader.equals("")) &&(env.getConfig("seedFTPPassword","").length() > 0) &&(env.getConfig("seedFilePath", "").length() > 0))) {enabledUploader = "Ftp";env.setConfig("seedUploadMethod",enabledUploader);- }-+ }+final HashMap<String, String> uploaders = Network.getSeedUploadMethods();prop.put("seedUploadMethods", uploaders.size() + 1);prop.put("seedUploadMethods_0_name", "none");prop.put("seedUploadMethods_0_selected", enabledUploader.equals("none") ? "1" : "0");prop.put("seedUploadMethods_0_file", "");-+int count = 0;final Iterator<String> uploaderKeys = uploaders.keySet().iterator();while (uploaderKeys.hasNext()) {count++;final String uploaderName = uploaderKeys.next();prop.put("seedUploadMethods_" +count+ "_name", uploaderName);- prop.put("seedUploadMethods_" +count+ "_selected", uploaderName.equals(enabledUploader) ? "1" : "0");+ prop.put("seedUploadMethods_" +count+ "_selected", uploaderName.equals(enabledUploader) ? "1" : "0");prop.put("seedUploadMethods_" +count+ "_file", "Settings_Seed_Upload" + uploaderName + ".inc");-+final yacySeedUploader theUploader = Network.getSeedUploader(uploaderName);final String[] configOptions = theUploader.getConfigurationOptions();if (configOptions != null) {@@ -193,10 +195,10 @@ public final class Settings_p {}}}-+// general settingsprop.put("seedURL", sb.peers.mySeed().get(Seed.SEEDLISTURL, ""));-+/** Message forwarding configuration*/@@ -215,9 +217,9 @@ public final class Settings_p {prop.put("server.https", sb.getConfigBool("server.https", false));prop.put("server.https_port.ssl", sb.getConfig(SwitchboardConstants.SERVER_SSLPORT,"8443"));prop.put("port.shutdown", sb.getConfig(SwitchboardConstants.SERVER_SHUTDOWNPORT, "-1"));-+// Referrer Policy- final String metaPolicy = env.getConfig(SwitchboardConstants.REFERRER_META_POLICY,+ final String metaPolicy = env.getConfig(SwitchboardConstants.REFERRER_META_POLICY,SwitchboardConstants.REFERRER_META_POLICY_DEFAULT);prop.put("metaPolicyNoReferrerChecked", ReferrerPolicy.NO_REFERRER.getValue().equals(metaPolicy) ? 1 : 0);prop.put("metaPolicySameOriginChecked", ReferrerPolicy.SAME_ORIGIN.getValue().equals(metaPolicy) ? 1 : 0);@@ -235,14 +237,14 @@ public final class Settings_p {prop.put("metaPolicyCustom_checked", 1);prop.put("metaPolicyCustom_value", metaPolicy);}-- prop.put("searchResultNoReferrerChecked", env.getConfigBool(SwitchboardConstants.SEARCH_RESULT_NOREFERRER,++ prop.put("searchResultNoReferrerChecked", env.getConfigBool(SwitchboardConstants.SEARCH_RESULT_NOREFERRER,SwitchboardConstants.SEARCH_RESULT_NOREFERRER_DEFAULT) ? 1 : 0);-+// debug/analysis- prop.put("solrBinaryResponseChecked", env.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,+ prop.put("solrBinaryResponseChecked", env.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT) ? 1 : 0);-+// HTTP clientprop.put(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED,env.getConfigBool(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED,@@ -259,25 +261,25 @@ public final class Settings_p {jvmSniExtensionSetting != null);prop.put(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED + ".readonly_jvmSettingValue",jvmSniExtensionSetting);-- /* For easier user understanding, the following flags controlling data sources selection++ /* For easier user understanding, the following flags controlling data sources selection* are rendered in the UI as checkboxes corresponding to enabled value when ticked */prop.put("searchLocalDHTChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_OFF, false) ? 1 : 0);prop.put("searchLocalSolrChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false) ? 1 : 0);prop.put("searchRemoteDHTChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false) ? 1 : 0);prop.put("searchRemoteSolrChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false) ? 1 : 0);-+prop.put("searchTestLocalDHTChecked", env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, false) ? 1 : 0);prop.put("searchTestLocalSolrChecked", env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false) ? 1 : 0);-+prop.put("searchShowRankingChecked", env.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_RANKING, SwitchboardConstants.SEARCH_RESULT_SHOW_RANKING_DEFAULT) ? 1 : 0);-+prop.put(SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED,sb.getConfigBool(SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED,SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED_DEFAULT));-+// return rewrite propertiesreturn prop;}-+}diff --git a/htroot/Status.java b/source/net/yacy/htroot/Status.javasimilarity index 96%rename from htroot/Status.javarename to source/net/yacy/htroot/Status.javaindex 0899bb341..dede1e48e 100644--- a/htroot/Status.java+++ b/source/net/yacy/htroot/Status.java@@ -27,6 +27,8 @@// javac -classpath .:../Classes Status.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.lang.management.ManagementFactory;import java.lang.management.ThreadMXBean;import java.util.Date;@@ -142,7 +144,7 @@ public class Statusprop.put("unrestrictedLocalAccess", 1);if(sb.getConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, "").indexOf("ConfigAccounts_p.html", 0) < 0) {/* Encourage checking accounts config page to be sure that unrestricted local access is desired */- prop.put("warningUnrestrictedLocalAccess", true);+ prop.put("warningUnrestrictedLocalAccess", true);}}@@ -196,13 +198,13 @@ public class Statusprop.put("extPortFormat", "0");}- Set<String> ips = Domains.myPublicIPs();+ final Set<String> ips = Domains.myPublicIPs();prop.put("host", ips.toString());// ssl supportprop.put("sslSupport", sb.getConfig("keyStore", "").isEmpty() || !sb.getConfigBool("server.https", false) ? 0 : 1);if (sb.getConfigBool("server.https", false)) prop.put("sslSupport_sslPort", sb.getHttpServer().getSslPort());-+// proxy informationif ( sb.getConfigBool("remoteProxyUse", false) ) {prop.put("remoteProxy", "1");@@ -213,7 +215,7 @@ public class Status}prop.put("info_isTransparentProxy", sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "0" : "1");prop.put("info_proxyURL", sb.getConfigBool("proxyURL", false) ? "0" : "1");-+// peer informationString thisHash = "";final String thisName = sb.peers.mySeed().getName();@@ -321,7 +323,7 @@ public class Statusprop.put("freeDisk", Formatter.bytesToString(sb.observer.getUsableSpace()));prop.put("processors", WorkflowProcessor.availableCPU);prop.put("load", Memory.getSystemLoadAverage());- ThreadMXBean threadbean = ManagementFactory.getThreadMXBean();+ final ThreadMXBean threadbean = ManagementFactory.getThreadMXBean();prop.put("processesCurrentInclDaemon", threadbean.getThreadCount());prop.put("processesCurrentOnlyDaemon", threadbean.getDaemonThreadCount());prop.put("processesTotal", threadbean.getTotalStartedThreadCount());diff --git a/htroot/Steering.java b/source/net/yacy/htroot/Steering.javasimilarity index 96%rename from htroot/Steering.javarename to source/net/yacy/htroot/Steering.javaindex e83cb5460..f59116710 100644--- a/htroot/Steering.java+++ b/source/net/yacy/htroot/Steering.java@@ -27,6 +27,8 @@// javac -classpath .:../Classes SettingsAck_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import net.yacy.cora.protocol.Domains;@@ -45,10 +47,10 @@ public class Steering {public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch ss) {if (post == null || post.isEmpty() || ss == null) {final serverObjects prop = new serverObjects();-+/* For authenticated user only : acquire a transaction token to pass then to the Steering.html post action */if(ss != null && ((Switchboard) ss).verifyAuthentication(header)) {- /* YaCyDefaultServlet will detect it and then also fill the custom HTTP response header used by the JavaScript shutdown and restart actions+ /* YaCyDefaultServlet will detect it and then also fill the custom HTTP response header used by the JavaScript shutdown and restart actions* or any external API requesting tool */prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));/* Also add to the Steering.html page info block for eventual display of this page without parameter */@@ -56,7 +58,7 @@ public class Steering {} else {prop.authenticationRequired();}-+return prop;}@@ -115,7 +117,7 @@ public class Steering {return prop;}-+return prop;}diff --git a/htroot/Supporter.java b/source/net/yacy/htroot/Supporter.javasimilarity index 99%rename from htroot/Supporter.javarename to source/net/yacy/htroot/Supporter.javaindex 02f44197b..ca05c1e01 100644--- a/htroot/Supporter.java+++ b/source/net/yacy/htroot/Supporter.java@@ -25,6 +25,8 @@// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.net.MalformedURLException;import java.util.Date;import java.util.HashMap;diff --git a/htroot/Surftips.java b/source/net/yacy/htroot/Surftips.javasimilarity index 99%rename from htroot/Surftips.javarename to source/net/yacy/htroot/Surftips.javaindex 893a27a5d..47e220b38 100644--- a/htroot/Surftips.java+++ b/source/net/yacy/htroot/Surftips.java@@ -25,6 +25,8 @@// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.net.MalformedURLException;import java.util.Date;import java.util.HashMap;diff --git a/htroot/Table_API_p.java b/source/net/yacy/htroot/Table_API_p.javasimilarity index 95%rename from htroot/Table_API_p.javarename to source/net/yacy/htroot/Table_API_p.javaindex 24400f3a7..f8c7d1a13 100644--- a/htroot/Table_API_p.java+++ b/source/net/yacy/htroot/Table_API_p.java@@ -17,6 +17,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.IOException;import java.nio.charset.StandardCharsets;import java.text.DateFormat;@@ -59,9 +61,9 @@ public class Table_API_p {public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {final Switchboard sb = (Switchboard) env;final serverObjects prop = new serverObjects();-+final DateFormat dateFormat = GenericFormatter.newSimpleDateFormat();-+prop.put("showexec", 0);prop.put("showtable", 0);@@ -87,7 +89,7 @@ public class Table_API_p {if (post != null && post.containsKey("filter") && post.get("filter", "").length() > 0) {typefilter = Pattern.compile(post.get("filter", ".*"));}-+/* Applying JSON API convention for the sort parameter format (http://jsonapi.org/format/#fetching-sorting) */String sortParam = WorkTables.TABLE_API_COL_DATE_RECORDING;if(post != null) {@@ -112,38 +114,38 @@ public class Table_API_p {current_pk = post.get("current_pk", "");}if (post != null && scheduleeventaction && !current_pk.isEmpty()) {-+/* Check this is a valid transaction */TransactionManager.checkPostTransaction(header, post);-+try {- Tables.Row row = sb.tables.select(WorkTables.TABLE_API_NAME, current_pk.getBytes());+ final Tables.Row row = sb.tables.select(WorkTables.TABLE_API_NAME, current_pk.getBytes());if (row != null) {String action;-+// eventsif (post.containsKey("event_select_" + current_pk) && post.get("event_select_" + current_pk, "off").equals("on")) {row.put(WorkTables.TABLE_API_COL_APICALL_EVENT_KIND, "regular");row.put(WorkTables.TABLE_API_COL_APICALL_EVENT_ACTION, "startup");}-+if (post.containsKey("event_kind_" + current_pk) ) {if ("off".equals(action = post.get("event_kind_" + current_pk, "off"))) {row.put(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, "");}row.put(WorkTables.TABLE_API_COL_APICALL_EVENT_KIND, action);}-+if (post.containsKey("event_action_" + current_pk) ) {row.put(WorkTables.TABLE_API_COL_APICALL_EVENT_ACTION, post.get("event_action_" + current_pk, "startup"));}-+// schedulerif (post.containsKey("repeat_select_" + current_pk) && post.get("repeat_select_" + current_pk, "off").equals("on")) {row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 7);row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_UNIT, "days");}-+if (post.containsKey("repeat_time_" + current_pk) ) {if ("off".equals(action = post.get("repeat_time_" + current_pk, "off"))) {row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 0);@@ -151,10 +153,10 @@ public class Table_API_p {row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, Integer.parseInt(action));}}-+if (post.containsKey("repeat_unit_" + current_pk) ) {action = post.get("repeat_unit_" + current_pk, "seldays");- int time = row.get(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 1);+ final int time = row.get(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 1);row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_UNIT, action.substring(3));if (action.equals("selminutes") && time > 0 && time < 10) {row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 10);@@ -169,26 +171,26 @@ public class Table_API_p {row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 30);}}-+// switch scheduler off if event kind is 'regular'final String kind = row.get(WorkTables.TABLE_API_COL_APICALL_EVENT_KIND, "off");if ("regular".equals(kind)) row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 0);-+WorkTables.calculateAPIScheduler(row, false);sb.tables.update(WorkTables.TABLE_API_NAME, row);}} catch (final Throwable e) { ConcurrentLog.logException(e); }}-+/* Edition of scheduled next execution dates */final Map<String, String> invalidNextExecDateFormats = new HashMap<>();final Map<String, String> nextExecDatesBeforeNow = new HashMap<>();if (post != null && post.containsKey("submitNextExecDates")) {-+/* Check this is a valid transaction */TransactionManager.checkPostTransaction(header, post);--++final String dateNexExecPrefix = "date_next_exec_";final Date now = new Date();for (final Map.Entry<String, String> entry : post.entrySet()) {@@ -202,20 +204,20 @@ public class Table_API_p {final String dateNextExecStr = entry.getValue().trim();try {final Date dateNextExec = dateFormat.parse(dateNextExecStr);-- if(time != 0) { // Check there is effectively a schedule period on this row++ if(time != 0) { // Check there is effectively a schedule period on this rowif(dateNextExec.before(now)) {nextExecDatesBeforeNow.put(rowPkStr, dateNextExecStr);} else {row.put(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, dateNextExec);-+sb.tables.update(WorkTables.TABLE_API_NAME, row);}}} catch (final ParseException e) {invalidNextExecDateFormats.put(rowPkStr, dateNextExecStr);}-+}} catch (final IOException | SpaceExceededException e) {ConcurrentLog.logException(e);@@ -223,12 +225,12 @@ public class Table_API_p {}}}-+if (post != null && !post.get("deleterows", "").isEmpty()) {-+/* Check this is a valid transaction */TransactionManager.checkPostTransaction(header, post);-+for (final Map.Entry<String, String> entry : post.entrySet()) {if (entry.getValue().startsWith("mark_")) {try {@@ -239,32 +241,32 @@ public class Table_API_p {}}}-+if (post != null && !post.get("deleteold", "").isEmpty()) {-+/* Check this is a valid transaction */TransactionManager.checkPostTransaction(header, post);-- int days = post.getInt("deleteoldtime", 365);++ final int days = post.getInt("deleteoldtime", 365);try {- Iterator<Row> ri = sb.tables.iterator(WorkTables.TABLE_API_NAME);+ final Iterator<Row> ri = sb.tables.iterator(WorkTables.TABLE_API_NAME);Row row;- Date now = new Date();- Date limit = new Date(now.getTime() - AbstractFormatter.dayMillis * days);- List<byte[]> pkl = new ArrayList<byte[]>();+ final Date now = new Date();+ final Date limit = new Date(now.getTime() - AbstractFormatter.dayMillis * days);+ final List<byte[]> pkl = new ArrayList<byte[]>();while (ri.hasNext()) {row = ri.next();- Date d = row.get(WorkTables.TABLE_API_COL_DATE_RECORDING, now);+ final Date d = row.get(WorkTables.TABLE_API_COL_DATE_RECORDING, now);if (d.before(limit)) {pkl.add(row.getPK());}}- for (byte[] pk: pkl) {+ for (final byte[] pk: pkl) {sb.tables.delete(WorkTables.TABLE_API_NAME, pk);}-+// store this call as api call; clean the call a bit before- Iterator<Entry<String, String[]>> ei = post.getSolrParams().getMap().entrySet().iterator();+ final Iterator<Entry<String, String[]>> ei = post.getSolrParams().getMap().entrySet().iterator();Entry<String, String[]> entry;while (ei.hasNext()) {entry = ei.next();@@ -282,10 +284,10 @@ public class Table_API_p {}if (post != null && !post.get("execrows", "").isEmpty()) {-+/* Check this is a valid transaction */TransactionManager.checkPostTransaction(header, post);-+// create a time-ordered list of events to executefinal Set<String> pks = new TreeSet<String>();for (final Map.Entry<String, String> entry : post.entrySet()) {@@ -317,16 +319,16 @@ public class Table_API_p {}prop.put("showexec_list", count);}-+// generate tableprop.put("showtable", 1);prop.put("showtable_inline", inline ? 1 : 0);-+/* Acquire a transaction token for the next POST form submission */final String nextTransactionToken = TransactionManager.getTransactionToken(header);prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken);prop.put("showtable_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken);-+final Date now = new Date();// insert rows@@ -361,7 +363,7 @@ public class Table_API_p {String type, comment, url;final boolean hasFilter = typefilter != QueryParams.catchall_pattern|| query != QueryParams.catchall_pattern;-+// first prepare a listwhile (mapIterator.hasNext()) {r = mapIterator.next();@@ -409,7 +411,7 @@ public class Table_API_p {} else {filteredSize = tablesize;}-+// then work on the listfor (final Tables.Row row : table) {final String rowPKStr = UTF8.String(row.getPK());@@ -427,24 +429,24 @@ public class Table_API_p {prop.put("showtable_list_" + count + "_callcount", callcount);prop.put("showtable_list_" + count + "_dateRecording", date_recording == null ? "-" : dateFormat.format(date_recording));prop.put("showtable_list_" + count + "_dateLastExec", date_last_exec == null ? "-" : dateFormat.format(date_last_exec));-+prop.put("showtable_list_" + count + "_editableDateNext", time != 0);final String enteredDateBeforeNow = nextExecDatesBeforeNow.get(rowPKStr);prop.put("showtable_list_" + count + "_editableDateNext_dateBeforeNowError", enteredDateBeforeNow != null);if(enteredDateBeforeNow != null) {prop.put("showtable_list_" + count + "_editableDateNext_dateBeforeNowError_invalidDate", enteredDateBeforeNow);}-+final String invalidEnteredDate = invalidNextExecDateFormats.get(rowPKStr);prop.put("showtable_list_" + count + "_editableDateNext_dateFormatError", invalidEnteredDate != null);if(invalidEnteredDate != null) {prop.put("showtable_list_" + count + "_editableDateNext_dateFormatError_invalidDate", invalidEnteredDate);}-+prop.put("showtable_list_" + count + "_editableDateNext_dateLastExecPattern", GenericFormatter.PATTERN_SIMPLE_REGEX);prop.put("showtable_list_" + count + "_editableDateNext_dateNextExec", date_next_exec == null ? "-" : dateFormat.format(date_next_exec));prop.put("showtable_list_" + count + "_editableDateNext_pk", rowPKStr);-+prop.put("showtable_list_" + count + "_type", row.get(WorkTables.TABLE_API_COL_TYPE));prop.putHTML("showtable_list_" + count + "_comment", row.get(WorkTables.TABLE_API_COL_COMMENT));// check type & action to link crawl start URLs back to CrawlStartExpert.html@@ -452,13 +454,13 @@ public class Table_API_p {&& prop.get("showtable_list_" + count + "_comment", "").startsWith("crawl start for")) {final String editUrl = UTF8.String(row.get(WorkTables.TABLE_API_COL_URL)).replace("Crawler_p", "CrawlStartExpert");if (editUrl.length() > 1000) {- final MultiProtocolURL u = new MultiProtocolURL("http://localhost:8090" + editUrl);+ final MultiProtocolURL u = new MultiProtocolURL("http://localhost:8090" + editUrl);prop.put("showtable_list_" + count + "_isCrawlerStart", 2);prop.put("showtable_list_" + count + "_isCrawlerStart_pk", rowPKStr);prop.put("showtable_list_" + count + "_isCrawlerStart_servlet", "CrawlStartExpert.html");- Map<String, String> attr = u.getAttributes();+ final Map<String, String> attr = u.getAttributes();int ac = 0;- for (Map.Entry<String, String> entry: attr.entrySet()) {+ for (final Map.Entry<String, String> entry: attr.entrySet()) {prop.put("showtable_list_" + count + "_isCrawlerStart_attr_" + ac + "_key", entry.getKey());prop.put("showtable_list_" + count + "_isCrawlerStart_attr_" + ac + "_value", entry.getValue());ac++;@@ -467,7 +469,7 @@ public class Table_API_p {} else {// short callsprop.put("showtable_list_" + count + "_isCrawlerStart", 1);- /* For better integration of YaCy peers behind a reverse proxy subfolder,+ /* For better integration of YaCy peers behind a reverse proxy subfolder,* ensure a path relative to this servlet (with no starting slash) is used for clone links.* We keep the paths starting with a slash for other URL displays. */prop.put("showtable_list_" + count + "_isCrawlerStart_url", editUrl.startsWith("/") ? editUrl.substring(1, editUrl.length()) : editUrl);@@ -481,12 +483,12 @@ public class Table_API_p {prop.put("showtable_list_" + count + "_scheduler_query", query.pattern());prop.put("showtable_list_" + count + "_scheduler_startRecord", startRecord);prop.put("showtable_list_" + count + "_scheduler_maximumRecords", maximumRecords);-+// eventsfinal String kind = row.get(WorkTables.TABLE_API_COL_APICALL_EVENT_KIND, "off");final String action = row.get(WorkTables.TABLE_API_COL_APICALL_EVENT_ACTION, "startup");prop.put("showtable_list_" + count + "_event_pk", rowPKStr);- boolean schedulerDisabled = "regular".equals(kind);+ final boolean schedulerDisabled = "regular".equals(kind);if ("off".equals(kind)) {prop.put("showtable_list_" + count + "_event", 0);} else {@@ -545,7 +547,7 @@ public class Table_API_p {prop.put("showtable_list_" + count + "_scheduler_scale", 31);prop.put("showtable_list_" + count + "_scheduler_selectedDays", 1);}-+}count++;}@@ -569,9 +571,9 @@ public class Table_API_p {prop.put("showtable_filter", typefilter.pattern());prop.put("showtable_query", queryParam);prop.put("showtable_sort", sortParam);-+putTableSortProperties(prop, sortDir, sortColumn);-+if (filteredSize > maximumRecords) {prop.put("showtable_navigation", 1);prop.put("showtable_navigation_startRecord", startRecord);@@ -603,7 +605,7 @@ public class Table_API_p {/*** Fill the serverObjects instance with table columns sort properties.- *+ ** @param prop* the serverObjects instance to fill. Must not be null.* @param sortDirdiff --git a/htroot/Table_RobotsTxt_p.java b/source/net/yacy/htroot/Table_RobotsTxt_p.javasimilarity index 91%rename from htroot/Table_RobotsTxt_p.javarename to source/net/yacy/htroot/Table_RobotsTxt_p.javaindex edbfd9ebf..6be9ac448 100644--- a/htroot/Table_RobotsTxt_p.java+++ b/source/net/yacy/htroot/Table_RobotsTxt_p.java@@ -1,3 +1,5 @@+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;diff --git a/htroot/Tables_p.java b/source/net/yacy/htroot/Tables_p.javasimilarity index 96%rename from htroot/Tables_p.javarename to source/net/yacy/htroot/Tables_p.javaindex eaf2f5af1..c9a11016f 100644--- a/htroot/Tables_p.java+++ b/source/net/yacy/htroot/Tables_p.java@@ -17,6 +17,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.IOException;import java.util.ArrayList;import java.util.HashMap;@@ -44,7 +46,7 @@ public class Tables_p {final String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, "");//final boolean json = ext.equals("json");final boolean xml = ext.equals("xml");-+prop.put("showtable", 0);prop.put("showedit", 0);prop.put("showselection", 0);@@ -172,7 +174,7 @@ public class Tables_p {prop.putXML("showtable_" + count + "_table", table); // only used in XMLbyte[] cell;while (plainIterator.hasNext() && count < maxcount) {- Tables.Row row = plainIterator.next();+ final Tables.Row row = plainIterator.next();if (row == null) continue;// write table contentdiff --git a/htroot/Threaddump_p.java b/source/net/yacy/htroot/Threaddump_p.javasimilarity index 86%rename from htroot/Threaddump_p.javarename to source/net/yacy/htroot/Threaddump_p.javaindex de52b3fd9..58b5b4390 100644--- a/htroot/Threaddump_p.java+++ b/source/net/yacy/htroot/Threaddump_p.java@@ -25,6 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.kelondro.logging.ThreadDump;import net.yacy.search.Switchboard;@@ -35,8 +37,8 @@ public class Threaddump_p {public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {- serverObjects prop = new serverObjects();- Switchboard sb = (Switchboard) env;+ final serverObjects prop = new serverObjects();+ final Switchboard sb = (Switchboard) env;final boolean plain = post != null && post.getBoolean("plain");final int sleep = (post == null) ? 0 : post.getInt("sleep", 0); // a sleep before creation of a thread dump can be used for profiling@@ -44,12 +46,12 @@ public class Threaddump_p {prop.put("dump", "1");int multipleCount = 100;- boolean multiple = post != null && post.containsKey("multipleThreaddump");+ final boolean multiple = post != null && post.containsKey("multipleThreaddump");if (multiple) {multipleCount = post.getInt("count", multipleCount);}- String threaddump = ThreadDump.threaddump(sb, plain, sleep, multiple, multipleCount);+ final String threaddump = ThreadDump.threaddump(sb, plain, sleep, multiple, multipleCount);prop.put("plain_count", multipleCount);prop.put("plain_content", threaddump);prop.put("plain", (plain) ? 1 : 0);diff --git a/htroot/Trails.java b/source/net/yacy/htroot/Trails.javasimilarity index 95%rename from htroot/Trails.javarename to source/net/yacy/htroot/Trails.javaindex 1d0fe663f..77af2d602 100644--- a/htroot/Trails.java+++ b/source/net/yacy/htroot/Trails.java@@ -7,17 +7,19 @@* modify it under the terms of the GNU Lesser General Public* License as published by the Free Software Foundation; either* version 2.1 of the License, or (at your option) any later version.- *+ ** This library is distributed in the hope that it will be useful,* but WITHOUT ANY WARRANTY; without even the implied warranty of* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU* Lesser General Public License for more details.- *+ ** You should have received a copy of the GNU Lesser General Public License* along with this program in the file lgpl21.txt* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;diff --git a/htroot/TransNews_p.java b/source/net/yacy/htroot/TransNews_p.javasimilarity index 89%rename from htroot/TransNews_p.javarename to source/net/yacy/htroot/TransNews_p.javaindex 323a1f004..c0652d855 100644--- a/htroot/TransNews_p.java+++ b/source/net/yacy/htroot/TransNews_p.java@@ -25,6 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.util.HashMap;@@ -49,7 +51,7 @@ public class TransNews_p {final Switchboard sb = (Switchboard) env;final serverObjects prop = new serverObjects();- String currentlang = sb.getConfig("locale.language", "default");+ final String currentlang = sb.getConfig("locale.language", "default");prop.put("currentlang", currentlang);if ("default".equals(currentlang) || "browser".equals(currentlang)) {@@ -60,38 +62,38 @@ public class TransNews_p {prop.put("errmsg", 0);TranslationManager transMgr = new TranslationManager();- File locallangFile = transMgr.getScratchFile(new File(currentlang + ".lng"));- Map<String, Map<String, String>> localTrans = transMgr.loadTranslationsLists(locallangFile);+ final File locallangFile = transMgr.getScratchFile(new File(currentlang + ".lng"));+ final Map<String, Map<String, String>> localTrans = transMgr.loadTranslationsLists(locallangFile);// calculate size of local translations listint size = 0;- for (Map<String, String> lst : localTrans.values()) {+ for (final Map<String, String> lst : localTrans.values()) {size += lst.size();}prop.put("transsize", size);-+// read votingif ((post != null) && post.containsKey("publishtranslation")) {- Iterator<String> filenameit = localTrans.keySet().iterator();+ final Iterator<String> filenameit = localTrans.keySet().iterator();int msgcounter = 0;while (filenameit.hasNext()) {- String file = filenameit.next();- Map<String, String> tmptrans = localTrans.get(file);- for (String sourcetxt : tmptrans.keySet()) {- String targettxt = tmptrans.get(sourcetxt);+ final String file = filenameit.next();+ final Map<String, String> tmptrans = localTrans.get(file);+ for (final String sourcetxt : tmptrans.keySet()) {+ final String targettxt = tmptrans.get(sourcetxt);if (targettxt != null && !targettxt.isEmpty()) {boolean sendit = true;// check if already published (in newsPool)- Iterator<NewsDB.Record> it = sb.peers.newsPool.recordIterator(NewsPool.INCOMING_DB);+ final Iterator<NewsDB.Record> it = sb.peers.newsPool.recordIterator(NewsPool.INCOMING_DB);while (it.hasNext()) {- NewsDB.Record rtmp = it.next();+ final NewsDB.Record rtmp = it.next();if (rtmp == null) {continue;}if (NewsPool.CATEGORY_TRANSLATION_ADD.equals(rtmp.category())) {- String tmplng = rtmp.attribute("language", null);- String tmpfile = rtmp.attribute("file", null);- String tmpsource = rtmp.attribute("source", null);+ final String tmplng = rtmp.attribute("language", null);+ final String tmpfile = rtmp.attribute("file", null);+ final String tmpsource = rtmp.attribute("source", null);//String tmptarget = rtmp.attribute("target", null);// if news with file and source exist (maybe from other peer) - skip sending another msg (to avoid confusion)@@ -135,12 +137,12 @@ public class TransNews_p {}}}-+if ((post != null) && ((refid = post.get("votePositive", null)) != null)) {final String filename = post.get("filename");- File lngfile = new File(sb.getAppPath("locale.source", "locales"), currentlang + ".lng");+ final File lngfile = new File(sb.getAppPath("locale.source", "locales"), currentlang + ".lng");transMgr = new TranslationManager(lngfile); // load full language for check if entry is new (globally)if (transMgr.addTranslation(filename, post.get("source"), post.get("target"))) {// add to local translation extension@@ -162,7 +164,7 @@ public class TransNews_p {sb.peers.newsPool.moveOff(NewsPool.INCOMING_DB, refid);} catch (IOException | SpaceExceededException ex) {}-+}// create Translation voting list@@ -182,7 +184,7 @@ public class TransNews_p {String target;while (k.hasNext()) {-+refid = k.next();if (refid == null) {continue;@@ -193,7 +195,7 @@ public class TransNews_p {continue;}- String lang = row.attribute("language", null);+ final String lang = row.attribute("language", null);filename = row.attribute("file", null);source = row.attribute("source", null);target = row.attribute("target", null);@@ -202,14 +204,14 @@ public class TransNews_p {}if (!lang.equals(currentlang)) continue;-+String existingtarget = null; //transMgr.getTranslation(filename, source);- Map<String, String> tmpMap = localTrans.get(filename);+ final Map<String, String> tmpMap = localTrans.get(filename);if (tmpMap != null) {existingtarget = tmpMap.get(source);}- boolean altexist = existingtarget != null && !target.isEmpty() && !existingtarget.isEmpty() && !existingtarget.equals(target);+ final boolean altexist = existingtarget != null && !target.isEmpty() && !existingtarget.isEmpty() && !existingtarget.equals(target);prop.put("results_" + i + "_refid", refid);prop.put("results_" + i + "_url", filename); // url to local file@@ -280,7 +282,7 @@ public class TransNews_p {String filename="";String source="";String target="";-+int score = 0;Integer vote;@@ -304,7 +306,7 @@ public class TransNews_p {}// add/subtract votes and write record-+if ((vote = negativeHashes.get(refid)) != null) {score -= vote.intValue();}diff --git a/htroot/Translator_p.java b/source/net/yacy/htroot/Translator_p.javasimilarity index 85%rename from htroot/Translator_p.javarename to source/net/yacy/htroot/Translator_p.javaindex e8af97b26..8a7906a4e 100644--- a/htroot/Translator_p.java+++ b/source/net/yacy/htroot/Translator_p.java@@ -18,10 +18,13 @@* along with this program in the file lgpl21.txt* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.util.Iterator;import java.util.Map;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.cora.util.ConcurrentLog;import net.yacy.data.TransactionManager;@@ -39,7 +42,7 @@ public class Translator_p {final servletProperties prop = new servletProperties();final Switchboard sb = (Switchboard) env;- String langcfg = sb.getConfig("locale.language", "default");+ final String langcfg = sb.getConfig("locale.language", "default");prop.put("targetlang", langcfg);if ("default".equals(langcfg) || "browser".equals(langcfg)) {prop.put("errmsg", 1); // msg: activate diff lng@@ -48,14 +51,14 @@ public class Translator_p {prop.put("errmsg", 0);final File localesFolder = sb.getAppPath("locale.source", "locales");- File lngfile = new File(localesFolder, langcfg + ".lng");- TranslationManager localTransMgr = new TranslationManager(/*new File ("locales","master.lng.xlf")*/);+ final File lngfile = new File(localesFolder, langcfg + ".lng");+ final TranslationManager localTransMgr = new TranslationManager(/*new File ("locales","master.lng.xlf")*/);- File masterxlf = new File(localesFolder, "master.lng.xlf");+ final File masterxlf = new File(localesFolder, "master.lng.xlf");if (!masterxlf.exists()) localTransMgr.createMasterTranslationLists(localesFolder, masterxlf);- Map<String, Map<String, String>> origTrans = localTransMgr.joinMasterTranslationLists(masterxlf, lngfile);+ final Map<String, Map<String, String>> origTrans = localTransMgr.joinMasterTranslationLists(masterxlf, lngfile);final File locallngfile = localTransMgr.getScratchFile(lngfile);- Map<String, Map<String, String>> localTrans = localTransMgr.loadTranslationsLists(locallngfile); // TODO: this will read file twice+ final Map<String, Map<String, String>> localTrans = localTransMgr.loadTranslationsLists(locallngfile); // TODO: this will read file twiceint i = 0;if (origTrans.size() > 0) {String filename = origTrans.keySet().iterator().next();@@ -64,14 +67,14 @@ public class Translator_p {}// prepare filename list for drop-down list- Iterator<String> filenameit = origTrans.keySet().iterator();+ final Iterator<String> filenameit = origTrans.keySet().iterator();final boolean filteruntranslated = post == null ? false : post.getBoolean("filteruntranslated");while (filenameit.hasNext()) {- String tmp = filenameit.next();+ final String tmp = filenameit.next();if (filteruntranslated) { // filter filenames with untranslated entries- Map<String, String> tmptrans = origTrans.get(tmp);+ final Map<String, String> tmptrans = origTrans.get(tmp);boolean hasuntrans = false;- for (String x : tmptrans.values()) {+ for (final String x : tmptrans.values()) {if (x == null || x.isEmpty()) {hasuntrans = true;break;@@ -86,7 +89,7 @@ public class Translator_p {prop.put("filelist", i);prop.add("sourcefile", filename);- Map<String, String> origTextList = origTrans.get(filename);+ final Map<String, String> origTextList = origTrans.get(filename);i = 0;boolean editapproved = false; // to switch already approved translation in edit mode@@ -100,12 +103,12 @@ public class Translator_p {}}boolean changed = false;- for (String sourcetext : origTextList.keySet()) {+ for (final String sourcetext : origTextList.keySet()) {String targettxt = origTextList.get(sourcetext);boolean existinlocalTrans = localTrans.containsKey(filename) && localTrans.get(filename).containsKey(sourcetext);-+if (editapproved) existinlocalTrans |= (i == textlistid); // if edit of approved requested handle as/like existing in local to allow edit-+if (targettxt == null || targettxt.isEmpty() || existinlocalTrans) {prop.put("textlist_" + i + "_filteruntranslated", true);} else if (filteruntranslated) {@@ -115,7 +118,7 @@ public class Translator_p {if (i == textlistid && post != null) {/* Check this is a valid transaction */TransactionManager.checkPostTransaction(requestHeader, post);-+if (editapproved) { // switch already translated in edit mode by copying to local translation// not saved here as not yet modified/approvedlocalTransMgr.addTranslation(localTrans, filename, sourcetext, targettxt);@@ -144,7 +147,7 @@ public class Translator_p {if (changed) {/* Check this is a valid transaction */TransactionManager.checkPostTransaction(requestHeader, post);-+localTransMgr.saveAsLngFile(langcfg, locallngfile, localTrans);// adhoc translate this file// 1. get/calc the path@@ -157,13 +160,13 @@ public class Translator_p {localTransMgr.translateFile(sourceFile, destFile, origTextList); // do the translation}}-+/* Acquire a transaction token for the next POST form submission */prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(requestHeader));-+prop.put("textlist", i);return prop;- } catch (IOException ex) {+ } catch (final IOException ex) {ConcurrentLog.logException(ex);}return null;diff --git a/htroot/User.java b/source/net/yacy/htroot/User.javasimilarity index 98%rename from htroot/User.javarename to source/net/yacy/htroot/User.javaindex 5f6fb9daa..5c8e3f130 100644--- a/htroot/User.java+++ b/source/net/yacy/htroot/User.java@@ -27,7 +27,10 @@//javac -classpath .:../Classes Message.java//if the shell's current path is HTROOT+package net.yacy.htroot;+import javax.servlet.ServletException;+import net.yacy.cora.order.Base64Order;import net.yacy.cora.order.Digest;import net.yacy.cora.protocol.RequestHeader;@@ -163,7 +166,7 @@ public class User{}try {requestHeader.logout(); // servlet container session logout- } catch (ServletException ex) {}+ } catch (final ServletException ex) {}if(post.containsKey("returnto")){prop.putHTML(serverObjects.ACTION_LOCATION, post.get("returnto"));}diff --git a/htroot/ViewFavicon.java b/source/net/yacy/htroot/ViewFavicon.javasimilarity index 92%rename from htroot/ViewFavicon.javarename to source/net/yacy/htroot/ViewFavicon.javaindex 492a5cb3a..755aa81f0 100644--- a/htroot/ViewFavicon.java+++ b/source/net/yacy/htroot/ViewFavicon.java@@ -18,6 +18,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.ByteArrayInputStream;import java.io.File;import java.io.IOException;@@ -45,12 +47,12 @@ import net.yacy.visualization.ImageViewer;/*** Extends ViewImage behavior : add a specific favicon cache and use of a* default image on loading error.- *+ ** @author luc**/public class ViewFavicon {-+/** Single instance of ImageViewer */private static final ImageViewer VIEWER = new ImageViewer();@@ -72,7 +74,7 @@ public class ViewFavicon {* "code" parameter (non authenticated users). When image could be parsed,* try encoding to target format specified by header "EXT". When any error* occurs, return default icon.- *+ ** @param header* request header* @param post@@ -85,8 +87,8 @@ public class ViewFavicon {public static Object respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {final Switchboard sb = (Switchboard) env;- String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, null);- boolean isPNGTarget = "png".equalsIgnoreCase(ext);+ final String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, null);+ final boolean isPNGTarget = "png".equalsIgnoreCase(ext);ImageInputStream imageInStream = null;InputStream inStream = null;@@ -99,7 +101,7 @@ public class ViewFavicon {final boolean auth = ImageViewer.hasFullViewingRights(header, sb); // handle access rights- DigestURL url = VIEWER.parseURL(post, auth);+ final DigestURL url = VIEWER.parseURL(post, auth);final String normalizedURL = url.toNormalform(false);@@ -108,7 +110,7 @@ public class ViewFavicon {}/* Icon is not already in cache */if (resultBytes == null) {- String urlExt = MultiProtocolURL.getFileExtension(url.getFileName());+ final String urlExt = MultiProtocolURL.getFileExtension(url.getFileName());if (ext != null && ext.equalsIgnoreCase(urlExt) && ImageViewer.isBrowserRendered(urlExt)) {return VIEWER.openInputStream(post, sb.loader, auth, url);}@@ -123,7 +125,7 @@ public class ViewFavicon {imageInStream = ImageIO.createImageInputStream(inStream);}// read image- EncodedImage encodedIcon = VIEWER.parseAndScale(post, auth, url, ext, imageInStream);+ final EncodedImage encodedIcon = VIEWER.parseAndScale(post, auth, url, ext, imageInStream);if (encodedIcon != null && !encodedIcon.getImage().isEmpty()) {resultBytes = encodedIcon.getImage().getBytes();@@ -133,7 +135,7 @@ public class ViewFavicon {}}}- } catch (IOException e) {+ } catch (final IOException e) {ConcurrentLog.fine("ViewFavicon", "Error loading favicon, default one wille be used : " + e);} finally {/*@@ -142,7 +144,7 @@ public class ViewFavicon {if (inStream != null) {try {inStream.close();- } catch (IOException ignored) {+ } catch (final IOException ignored) {}}if (resultBytes == null) {@@ -167,7 +169,7 @@ public class ViewFavicon {/*** Load default icon and encode it to ext format- *+ ** @param post* post parameters* @param sb@@ -177,7 +179,7 @@ public class ViewFavicon {* @return icon encoded bytes, empty if and exception occured when loading* or rendering*/- private static byte[] loadDefaultIcon(final serverObjects post, final Switchboard sb, String ext) {+ private static byte[] loadDefaultIcon(final serverObjects post, final Switchboard sb, final String ext) {byte[] resultBytes;byte[] defaultBytes = new byte[0];try {diff --git a/htroot/ViewFile.java b/source/net/yacy/htroot/ViewFile.javasimilarity index 96%rename from htroot/ViewFile.javarename to source/net/yacy/htroot/ViewFile.javaindex e8a4de344..bbf1ba1ee 100644--- a/htroot/ViewFile.java+++ b/source/net/yacy/htroot/ViewFile.java@@ -27,6 +27,8 @@//javac -classpath .:../Classes Status.java//if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.IOException;import java.net.MalformedURLException;import java.nio.charset.Charset;@@ -95,11 +97,11 @@ public class ViewFile {prop.put("error", "1");prop.put("url", "");prop.put("showSnippet", 0);-+if (post == null) return prop;// get segment- Segment indexSegment = sb.index;+ final Segment indexSegment = sb.index;final boolean authorized = sb.verifyAuthentication(header);if (post.containsKey("words"))@@ -120,7 +122,7 @@ public class ViewFile {final String viewMode = showSnippet ? "sentences" : post.get("viewMode", "sentences");prop.put("error_vMode-" + viewMode, "1");prop.put("viewModeValue", viewMode);-+DigestURL url = null;String descr = "";final int wordCount = 0;@@ -216,9 +218,9 @@ public class ViewFile {/* Encoding is known but not supported on this system : default decode using UTF-8 */charsetName = StandardCharsets.UTF_8.name();}- } catch(IllegalCharsetNameException e) {+ } catch(final IllegalCharsetNameException e) {/* Encoding is known but charset name is not valid : default decode using UTF-8 */- charsetName = StandardCharsets.UTF_8.name();+ charsetName = StandardCharsets.UTF_8.name();}content = new String(response.getContent(), charsetName);} catch (final Exception e) {@@ -346,8 +348,8 @@ public class ViewFile {}// optional: generate snippetif (showSnippet) {- QueryGoal goal = new QueryGoal(post.get("search", ""));- TextSnippet snippet = new TextSnippet(+ final QueryGoal goal = new QueryGoal(post.get("search", ""));+ final TextSnippet snippet = new TextSnippet(sb.loader,urlEntry,goal.getIncludeWordsSet(),@@ -383,7 +385,7 @@ public class ViewFile {prop.put("error_inurldb_hash", urlHash);prop.put("error_wordCount", wordCount);prop.put("error_firstSeen", "");- long firstseen = sb.index.getFirstSeenTime(ASCII.getBytes(urlHash));+ final long firstseen = sb.index.getFirstSeenTime(ASCII.getBytes(urlHash));prop.put("error_firstSeen", firstseen < 0 ? "" : new Date(firstseen).toString());prop.putHTML("error_desc", (descr.isEmpty()) ? " " : descr);prop.putNum("error_size", size);@@ -566,7 +568,7 @@ public class ViewFile {}return i;}-+/*** Fill prop object with icon links.* @param prop object ot be filleddiff --git a/htroot/ViewImage.java b/source/net/yacy/htroot/ViewImage.javasimilarity index 92%rename from htroot/ViewImage.javarename to source/net/yacy/htroot/ViewImage.javaindex 1db8970eb..1f3169c74 100644--- a/htroot/ViewImage.java+++ b/source/net/yacy/htroot/ViewImage.java@@ -21,6 +21,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.IOException;import java.io.InputStream;@@ -39,7 +41,7 @@ import net.yacy.server.serverSwitch;import net.yacy.visualization.ImageViewer;public class ViewImage {-+/** Single instance of ImageViewer */private static final ImageViewer VIEWER = new ImageViewer();@@ -48,7 +50,7 @@ public class ViewImage {* When image format is not supported, return directly image data. When* image could be parsed, try encoding to target format specified by header* "EXT".- *+ ** @param header* request header* @param post@@ -56,9 +58,9 @@ public class ViewImage {* @param env* environment* @return an {@link EncodedImage} instance encoded in format specified in- * post, or an InputStream pointing to original image data.- * Return and EncodedImage with empty data when image format is not supported,- * a read/write or any other error occured while loading resource.+ * post, or an InputStream pointing to original image data.+ * Return and EncodedImage with empty data when image format is not supported,+ * a read/write or any other error occured while loading resource.* @throws IOException* when specified url is malformed.* Sould end in a HTTP 500 error whose processing is more@@ -75,10 +77,10 @@ public class ViewImage {throw new TemplateMissingParameterException("please fill at least url or code parameter");}- String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, null);+ final String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, null);final boolean auth = ImageViewer.hasFullViewingRights(header, sb); // handle access rights- DigestURL url = VIEWER.parseURL(post, auth);+ final DigestURL url = VIEWER.parseURL(post, auth);// get the image as streamEncodedImage encodedImage;@@ -86,7 +88,7 @@ public class ViewImage {ImageInputStream imageInStream = null;InputStream inStream = null;try {- String urlExt = MultiProtocolURL.getFileExtension(url.getFileName());+ final String urlExt = MultiProtocolURL.getFileExtension(url.getFileName());if (ext != null && ext.equalsIgnoreCase(urlExt) && ImageViewer.isBrowserRendered(urlExt)) {return VIEWER.openInputStream(post, sb.loader, auth, url);}@@ -102,7 +104,7 @@ public class ViewImage {}// read imageencodedImage = VIEWER.parseAndScale(post, auth, url, ext, imageInStream);- } catch (Exception e) {+ } catch (final Exception e) {/** Exceptions are not propagated here : many error causes are* possible, network errors, incorrect or unsupported format, bad@@ -118,7 +120,7 @@ public class ViewImage {if (inStream != null) {try {inStream.close();- } catch (IOException ignored) {+ } catch (final IOException ignored) {}}}diff --git a/htroot/ViewLog_p.java b/source/net/yacy/htroot/ViewLog_p.javasimilarity index 96%rename from htroot/ViewLog_p.javarename to source/net/yacy/htroot/ViewLog_p.javaindex a5783226e..f0a2df9d2 100644--- a/htroot/ViewLog_p.java+++ b/source/net/yacy/htroot/ViewLog_p.java@@ -30,6 +30,8 @@//javac -classpath .:../classes ViewLog_p.java//if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.logging.Handler;import java.util.logging.Logger;import java.util.regex.Matcher;diff --git a/htroot/ViewProfile.java b/source/net/yacy/htroot/ViewProfile.javasimilarity index 95%rename from htroot/ViewProfile.javarename to source/net/yacy/htroot/ViewProfile.javaindex e1b1750b1..ab99b3f92 100644--- a/htroot/ViewProfile.java+++ b/source/net/yacy/htroot/ViewProfile.java@@ -31,6 +31,8 @@// javac -classpath .:../Classes Blacklist_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import java.io.FileInputStream;import java.io.IOException;@@ -67,7 +69,7 @@ public class ViewProfile {env.getConfig("network.unit.description", "") :env.getConfig(SwitchboardConstants.GREETING, "");prop.put("display_promoteSearchPageGreeting", promoteSearchPageGreeting);-+prop.put("edit", authenticated ? 1 : 0);final String hash = (post == null) ? null : post.get("hash");@@ -91,7 +93,7 @@ public class ViewProfile {if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (final Exception e) {}}profile = new HashMap<String, String>();- for (Map.Entry<Object, Object> e: p.entrySet()) profile.put((String) e.getKey(), (String) e.getValue());+ for (final Map.Entry<Object, Object> e: p.entrySet()) profile.put((String) e.getKey(), (String) e.getValue());prop.put("success", "3"); // everything okprop.put("localremotepeer", "0");prop.putHTML("success_peername", sb.peers.mySeed().getName());diff --git a/htroot/Vocabulary_p.java b/source/net/yacy/htroot/Vocabulary_p.javasimilarity index 90%rename from htroot/Vocabulary_p.javarename to source/net/yacy/htroot/Vocabulary_p.javaindex 5944c0fa0..614964eb0 100644--- a/htroot/Vocabulary_p.java+++ b/source/net/yacy/htroot/Vocabulary_p.java@@ -18,6 +18,8 @@* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import java.io.BufferedReader;import java.io.File;import java.io.FileNotFoundException;@@ -65,24 +67,24 @@ import net.yacy.server.serverSwitch;* Handle creation and edition of vocabularies through the Vocabulary_p.html page.*/public class Vocabulary_p {-+/** Logger */private final static ConcurrentLog LOG = new ConcurrentLog(Vocabulary_p.class.getSimpleName());public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {final Switchboard sb = (Switchboard) env;final serverObjects prop = new serverObjects();-+/* Acquire a transaction token for the next POST form submission */final String nextToken = TransactionManager.getTransactionToken(header);prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, nextToken);prop.put("edit_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextToken);prop.put("create_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextToken);-- Collection<Tagging> vocs = LibraryProvider.autotagging.getVocabularies();++ final Collection<Tagging> vocs = LibraryProvider.autotagging.getVocabularies();String vocabularyName = (post == null) ? null : post.get("vocabulary", null);- String discovername = (post == null) ? null : post.get("discovername", null);+ final String discovername = (post == null) ? null : post.get("discovername", null);Tagging vocabulary = vocabularyName == null ? null : LibraryProvider.autotagging.getVocabulary(vocabularyName);if (vocabulary == null) {vocabularyName = null;@@ -92,7 +94,7 @@ public class Vocabulary_p {if (vocabulary == null && discovername != null && discovername.length() > 0) {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);-+// get details of creationString discoverobjectspace = post.get("discoverobjectspace", "");MultiProtocolURL discoveruri = null;@@ -123,7 +125,7 @@ public class Vocabulary_p {csvUrl = new DigestURL(discoverFromCSVPath);} catch(final MalformedURLException e) {csvFileStatus = 5;- prop.put("create_csvFileStatus_csvUrl", discoverFromCSVPath);+ prop.put("create_csvFileStatus_csvUrl", discoverFromCSVPath);}} else {final File discoverFromCSVFile = new File(discoverFromCSVPath);@@ -142,31 +144,31 @@ public class Vocabulary_p {csvUrl = new DigestURL(discoverFromCSVFile);} catch(final MalformedURLException e) {csvFileStatus = 5;- prop.put("create_csvFileStatus_csvUrl", "file://" + discoverFromCSVFile.getAbsolutePath());+ prop.put("create_csvFileStatus_csvUrl", "file://" + discoverFromCSVFile.getAbsolutePath());}}}-+if(csvUrl != null) {try {handleDiscoverFromCSV(sb, post, table, csvUrl);} catch(final IOException e) {LOG.warn("Could not read CSV file at " + csvUrl, e);csvFileStatus = 3;- prop.put("create_csvFileStatus_csvFile", csvUrl.toString());+ prop.put("create_csvFileStatus_csvFile", csvUrl.toString());}}}} else {- Iterator<DigestURL> ui = segment.urlSelector(discoveruri, Long.MAX_VALUE, 100000);+ final Iterator<DigestURL> ui = segment.urlSelector(discoveruri, Long.MAX_VALUE, 100000);while (ui.hasNext()) {- DigestURL u = ui.next();- String u0 = u.toNormalform(true);+ final DigestURL u = ui.next();+ final String u0 = u.toNormalform(true);t = "";if (discoverFromPath) {- int exp = u0.lastIndexOf('.');+ final int exp = u0.lastIndexOf('.');if (exp < 0) continue;- int slp = u0.lastIndexOf('/', exp);+ final int slp = u0.lastIndexOf('/', exp);if (slp < 0) continue;t = u0.substring(slp, exp);int p;@@ -174,28 +176,28 @@ public class Vocabulary_p {while ((p = t.indexOf('=')) >= 0) t = t.substring(p + 1);}if (discoverFromTitle || discoverFromTitleSplitted) {- URIMetadataNode m = segment.fulltext().getMetadata(u.hash());+ final URIMetadataNode m = segment.fulltext().getMetadata(u.hash());if (m != null) t = m.dc_title();if (t.endsWith(".jpg") || t.endsWith(".gif")) continue;}if (discoverFromAuthor) {- URIMetadataNode m = segment.fulltext().getMetadata(u.hash());+ final URIMetadataNode m = segment.fulltext().getMetadata(u.hash());if (m != null) t = m.dc_creator();}t = t.replaceAll("_", " ").replaceAll("\"", " ").replaceAll("'", " ").replaceAll(",", " ").replaceAll(" ", " ").trim();if (t.isEmpty()) continue;if (discoverFromTitleSplitted) {- String[] ts = CommonPattern.SPACES.split(t);- for (String s: ts) {+ final String[] ts = CommonPattern.SPACES.split(t);+ for (final String s: ts) {if (s.isEmpty()) continue;if (s.endsWith(".jpg") || s.endsWith(".gif")) continue;table.put(s, new Tagging.SOTuple(Tagging.normalizeTerm(s), u0));}} else if (discoverFromAuthor) {- String[] ts = CommonPattern.SEMICOLON.split(t); // author names are often separated by ';'+ final String[] ts = CommonPattern.SEMICOLON.split(t); // author names are often separated by ';'for (String s: ts) {if (s.isEmpty()) continue;- int p = s.indexOf(','); // check if there is a reversed method to mention the name+ final int p = s.indexOf(','); // check if there is a reversed method to mention the nameif (p >= 0) s = s.substring(p + 1).trim() + " " + s.substring(0, p).trim();table.put(s, new Tagging.SOTuple(Tagging.normalizeTerm(s), u0));}@@ -208,13 +210,13 @@ public class Vocabulary_p {prop.put("create_csvFileStatus", csvFileStatus);if(csvFileStatus == 0) {try {- Tagging newvoc = new Tagging(discovername, propFile, discoverobjectspace, table);+ final Tagging newvoc = new Tagging(discovername, propFile, discoverobjectspace, table);prop.put("create_vocabWriteError", false);-+LibraryProvider.autotagging.addVocabulary(newvoc);vocabularyName = discovername;vocabulary = newvoc;-+// store this call as api callsb.tables.recordAPICall(post, "Vocabulary_p.html", WorkTables.TABLE_API_TYPE_CRAWLER, "vocabulary creation for " + discovername);} catch(final IOException e) {@@ -227,7 +229,7 @@ public class Vocabulary_p {} else if (vocabulary != null && post.containsKey("set")) {/* Check the transaction is valid */TransactionManager.checkPostTransaction(header, post);-+try {// check if objectspace was setvocabulary.setObjectspace(post.get("objectspace", vocabulary.getObjectspace() == null ? "" : vocabulary.getObjectspace()));@@ -242,11 +244,11 @@ public class Vocabulary_p {}// check if a term was modified- for (Map.Entry<String, String> e : post.entrySet()) {+ for (final Map.Entry<String, String> e : post.entrySet()) {if (e.getKey().startsWith("modify_") && e.getValue().equals("checked")) {- String term = e.getKey().substring(7);- String synonyms = post.get("synonyms_" + term, "");- String objectlink = post.get("objectlink_" + term, "");+ final String term = e.getKey().substring(7);+ final String synonyms = post.get("synonyms_" + term, "");+ final String objectlink = post.get("objectlink_" + term, "");vocabulary.put(term, synonyms, objectlink);}}@@ -264,32 +266,32 @@ public class Vocabulary_p {}// check if a term shall be deleted- if (vocabulary != null && vocabulary.size() > 0) for (Map.Entry<String, String> e : post.entrySet()) {+ if (vocabulary != null && vocabulary.size() > 0) for (final Map.Entry<String, String> e : post.entrySet()) {if (e.getKey().startsWith("delete_") && e.getValue().equals("checked")) {- String term = e.getKey().substring(7);+ final String term = e.getKey().substring(7);vocabulary.delete(term);}}-+// check the isFacet and isMatchFromLinkedData propertiesif (vocabulary != null && post.containsKey("set")) {- boolean isFacet = post.getBoolean("isFacet");+ final boolean isFacet = post.getBoolean("isFacet");vocabulary.setFacet(isFacet);- Set<String> omit = env.getConfigSet("search.result.show.vocabulary.omit");+ final Set<String> omit = env.getConfigSet("search.result.show.vocabulary.omit");if (isFacet) {- omit.remove(vocabularyName);+ omit.remove(vocabularyName);} else {omit.add(vocabularyName);}env.setConfig("search.result.show.vocabulary.omit", omit);-- boolean isMatchFromLinkedData = post.getBoolean("vocabularies.matchLinkedData");++ final boolean isMatchFromLinkedData = post.getBoolean("vocabularies.matchLinkedData");vocabulary.setMatchFromLinkedData(isMatchFromLinkedData);final Set<String> matchLinkedDataVocs = env.getConfigSet(SwitchboardConstants.VOCABULARIES_MATCH_LINKED_DATA_NAMES);if (isMatchFromLinkedData) {matchLinkedDataVocs.add(vocabularyName);} else {- matchLinkedDataVocs.remove(vocabularyName);+ matchLinkedDataVocs.remove(vocabularyName);}env.setConfig(SwitchboardConstants.VOCABULARIES_MATCH_LINKED_DATA_NAMES, matchLinkedDataVocs);}@@ -301,7 +303,7 @@ public class Vocabulary_p {}int count = 0;- for (Tagging v: vocs) {+ for (final Tagging v: vocs) {prop.put("vocabularyset_" + count + "_name", v.getName());prop.put("vocabularyset_" + count + "_selected", ((vocabularyName != null && vocabularyName.equals(v.getName())) || (discovername != null && discovername.equals(v.getName()))) ? 1 : 0);count++;@@ -314,7 +316,7 @@ public class Vocabulary_p {prop.put("edit", 0);} else {prop.put("edit", 1);- boolean editable = vocabulary.getFile() != null && vocabulary.getFile().exists();+ final boolean editable = vocabulary.getFile() != null && vocabulary.getFile().exists();prop.put("edit_editable", editable ? 1 : 0);prop.putHTML("edit_editable_file", editable ? vocabulary.getFile().getAbsolutePath() : "");prop.putHTML("edit_name", vocabulary.getName());@@ -329,20 +331,20 @@ public class Vocabulary_p {prop.putHTML("edit_editable_objectspacepredicate", DCTerms.references.getPredicate());int c = 0;boolean dark = false;- int osl = vocabulary.getObjectspace() == null ? 0 : vocabulary.getObjectspace().length();- Map<String, SOTuple> list = vocabulary.list();+ final int osl = vocabulary.getObjectspace() == null ? 0 : vocabulary.getObjectspace().length();+ final Map<String, SOTuple> list = vocabulary.list();prop.put("edit_size", list.size());- for (Map.Entry<String, SOTuple> entry: list.entrySet()) {+ for (final Map.Entry<String, SOTuple> entry: list.entrySet()) {prop.put("edit_terms_" + c + "_editable", editable ? 1 : 0);prop.put("edit_terms_" + c + "_dark", dark ? 1 : 0); dark = !dark;prop.putXML("edit_terms_" + c + "_label", osl > entry.getValue().getObjectlink().length() ? entry.getKey() : entry.getValue().getObjectlink().substring(osl));prop.putHTML("edit_terms_" + c + "_term", entry.getKey());prop.putXML("edit_terms_" + c + "_termxml", entry.getKey());prop.putHTML("edit_terms_" + c + "_editable_term", entry.getKey());- String synonymss = entry.getValue().getSynonymsCSV();+ final String synonymss = entry.getValue().getSynonymsCSV();prop.putHTML("edit_terms_" + c + "_editable_synonyms", synonymss);if (synonymss.length() > 0) {- String[] synonymsa = entry.getValue().getSynonymsList();+ final String[] synonymsa = entry.getValue().getSynonymsList();for (int i = 0; i < synonymsa.length; i++) {prop.put("edit_terms_" + c + "_synonyms_" + i + "_altLabel", synonymsa[i]);}@@ -362,22 +364,22 @@ public class Vocabulary_p {prop.putHTML("create_charset_" + 0 + "_name", "autodetect");prop.put("create_charset_" + 0 + "_selected", 1);int c = 1;- for (String cs: Charset.availableCharsets().keySet()) {+ for (final String cs: Charset.availableCharsets().keySet()) {prop.putHTML("create_charset_" + c + "_name", cs);prop.put("create_charset_" + c + "_selected", 0);c++;}prop.put("create_charset", c);-+// return rewrite propertiesreturn prop;}-+/*** Parse a CSV content line and extract field values. When the last field of* this line starts with and unclosed escape character, the current line is* appended to the escapedContent buffer.- *+ ** @param line* a raw line from a CSV document. Must not be null.* @param separatorPattern@@ -396,7 +398,7 @@ public class Vocabulary_p {if (tokens.length == 0) {tokens = new String[]{line};}-+/* Handle continuation of multi-lines field content escaped between escape char */if(multiLineContent.length() > 0) {int closingEscapeIndex = -1;@@ -420,7 +422,7 @@ public class Vocabulary_p {return fields;}}-+/* Handle separator char escaped between escape char */final StringBuilder escapedSeparatorContent = new StringBuilder();for(final String field : tokens) {@@ -443,7 +445,7 @@ public class Vocabulary_p {}fields.add(field);}-+if(escapedSeparatorContent.length() > 0) {/* Handle beginning of field content with escaped line separator */multiLineContent.setLength(0);@@ -474,9 +476,9 @@ public class Vocabulary_p {final int discovercolumnobjectlink = post.getInt("discovercolumnobjectlink", -1);final boolean discoverenrichsynonyms = post.get("discoversynonymsmethod", "none").equals("enrichsynonyms");final boolean discoverreadcolumn = post.get("discoversynonymsmethod", "none").equals("readcolumn");-+final Pattern separatorPattern = Pattern.compile(columnSeparator);-+// auto-detect charset, used code from http://jchardet.sourceforge.net/; see also: http://www-archive.mozilla.org/projects/intl/chardet.htmlif (charsetName.equals("autodetect")) {@@ -486,9 +488,9 @@ public class Vocabulary_p {if(streamResponse == null || streamResponse.getContentStream() == null) {throw new IOException("Could not get CSV content at " + csvFileUrl);}-+charsetName = streamResponse.getResponse().getCharacterEncoding();-+if(charsetName == null) {/* Charset not provided in response headers : try to detect it from content */final List<String> charsets = FileUtils.detectCharset(streamResponse.getContentStream());@@ -507,7 +509,7 @@ public class Vocabulary_p {}}}-+// when autodetection of content charset has been selected, a remote resource may opened again, but has some chances to be now in cachetry(final StreamResponse streamResponse = sb.loader.openInputStream(sb.loader.request(csvFileUrl, true, false), CacheStrategy.IFFRESH, BlacklistType.CRAWLER,@@ -527,8 +529,8 @@ public class Vocabulary_p {/*** Fill the vocabulary table from reader open on CSV content.* @param table the vocabulary table to fill. Must not be null.- * @param escapeChar CSV escape character (standard is double quote). Must not be null.- * @param lineStart index (zero based) of the first line to parse. Previous lines are ignored.+ * @param escapeChar CSV escape character (standard is double quote). Must not be null.+ * @param lineStart index (zero based) of the first line to parse. Previous lines are ignored.* @param literalsColumn index (zero based) of the column to read for literals* @param synonymsColumn index (zero based) of the column to read for synonyms. Set to -1 to ignore.* @param discovercolumnobjectlink@@ -554,13 +556,13 @@ public class Vocabulary_p {if (line.length() == 0) {continue;}-+final List<String> fields = parseCSVLine(line, separatorPattern, escapeChar, multiLineContent);if (multiLineContent.length() > 0) {continue;}--++String literal = literalsColumn < 0 || fields.size() <= literalsColumn ? null : fields.get(literalsColumn).trim();if (literal == null) {continue;@@ -584,11 +586,11 @@ public class Vocabulary_p {}// check double synonymsif (synonyms.length() > 0) {- String oldliteral = synonym2literal.get(synonyms);+ final String oldliteral = synonym2literal.get(synonyms);if (oldliteral != null && !literal.equals(oldliteral)) {// replace old entry with combined newtable.remove(oldliteral);- String newliteral = oldliteral + "," + literal;+ final String newliteral = oldliteral + "," + literal;literal = newliteral;}synonym2literal.put(synonyms, literal);@@ -598,7 +600,7 @@ public class Vocabulary_p {}}}-+private static String normalizeLiteral(String literal, final String escapeChar) {if (literal == null) {return "";diff --git a/htroot/WatchWebStructure_p.java b/source/net/yacy/htroot/WatchWebStructure_p.javasimilarity index 95%rename from htroot/WatchWebStructure_p.javarename to source/net/yacy/htroot/WatchWebStructure_p.javaindex 9a7fa087a..fa1f8b5bd 100644--- a/htroot/WatchWebStructure_p.java+++ b/source/net/yacy/htroot/WatchWebStructure_p.java@@ -4,6 +4,8 @@//$LastChangedBy$//+package net.yacy.htroot;+import java.util.Iterator;import net.yacy.cora.document.id.DigestURL;@@ -68,12 +70,12 @@ public class WatchWebStructure_p {host = "www." + host;}}-+if (post != null && post.containsKey("hosts")) {- int maxcount = 200;- ReversibleScoreMap<String> score = sb.webStructure.hostReferenceScore();+ final int maxcount = 200;+ final ReversibleScoreMap<String> score = sb.webStructure.hostReferenceScore();int c = 0;- Iterator<String> i = score.keys(false);+ final Iterator<String> i = score.keys(false);String h;while (i.hasNext() && c < maxcount) {h = i.next();diff --git a/htroot/WebStructurePicture_p.java b/source/net/yacy/htroot/WebStructurePicture_p.javasimilarity index 87%rename from htroot/WebStructurePicture_p.javarename to source/net/yacy/htroot/WebStructurePicture_p.javaindex 8efa8ace8..7e5ea9e38 100644--- a/htroot/WebStructurePicture_p.java+++ b/source/net/yacy/htroot/WebStructurePicture_p.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Date;import java.util.HashSet;import java.util.Iterator;@@ -88,7 +90,7 @@ public class WebStructurePicture_p {color_lineend = post.get("colorlineend", color_lineend);cyc = post.getInt("cyc", 0);}-+// calculate target timefinal long timeout = (time < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + (time * 8 / 10);@@ -107,9 +109,9 @@ public class WebStructurePicture_p {} else {// recursively find domains, up to a specific depthGraphPlotter graph = new GraphPlotter();- String[] hostlist = CommonPattern.COMMA.split(hosts);+ final String[] hostlist = CommonPattern.COMMA.split(hosts);for (int i = 0; i < hostlist.length; i++) {- String host = hostlist[i];+ final String host = hostlist[i];double angle = 2.0d * i * Math.PI / hostlist.length;if (hostlist.length == 3) angle -= Math.PI / 2;if (hostlist.length == 4) angle += Math.PI / 4;@@ -120,9 +122,9 @@ public class WebStructurePicture_p {// apply physics to it to get a better shapeif (post != null && post.containsKey("pa")) {// test with: http://localhost:8090/WebStructurePicture_p.png?pa=1&ral=0.7&raa=0.5&rar=2&rel=0.5&rea=1&rer=2- GraphPlotter.Ribbon rAll = new GraphPlotter.Ribbon(post.getFloat("ral", 0.1f), post.getFloat("raa", 0.1f), post.getFloat("rar", 0.1f));- GraphPlotter.Ribbon rEdge = new GraphPlotter.Ribbon(post.getFloat("rel", 0.05f), post.getFloat("rea", 0.1f), post.getFloat("rer", 0.1f));- int pa = post.getInt("pa", 0);+ final GraphPlotter.Ribbon rAll = new GraphPlotter.Ribbon(post.getFloat("ral", 0.1f), post.getFloat("raa", 0.1f), post.getFloat("rar", 0.1f));+ final GraphPlotter.Ribbon rEdge = new GraphPlotter.Ribbon(post.getFloat("rel", 0.05f), post.getFloat("rea", 0.1f), post.getFloat("rer", 0.1f));+ final int pa = post.getInt("pa", 0);for (int i = 0; i < pa; i++) graph = graph.physics(rAll, rEdge);}@@ -140,25 +142,25 @@ public class WebStructurePicture_p {}private static final int place(- final GraphPlotter graph, final WebStructureGraph structure, String hostName,- int bf, int maxnodes, final long timeout, int nextlayer, final int maxlayer, final int cyc) {- Point pivotpoint = graph.getNode(hostName);+ final GraphPlotter graph, final WebStructureGraph structure, final String hostName,+ final int bf, int maxnodes, final long timeout, int nextlayer, final int maxlayer, final int cyc) {+ final Point pivotpoint = graph.getNode(hostName);int branches = 0;if (nextlayer == maxlayer) return branches;nextlayer++;final double radius = 1.0 / (1 << nextlayer);final Map<String, Integer> next = structure.outgoingReferencesByHostName(hostName);- ClusteredScoreMap<String> next0 = new ClusteredScoreMap<String>(false);- for (Map.Entry<String, Integer> entry: next.entrySet()) next0.set(entry.getKey(), entry.getValue());+ final ClusteredScoreMap<String> next0 = new ClusteredScoreMap<String>(false);+ for (final Map.Entry<String, Integer> entry: next.entrySet()) next0.set(entry.getKey(), entry.getValue());// first set points to next hostsfinal Set<String> targetHostNames = new HashSet<String>();int maxtargetrefs = 8, maxthisrefs = 8;int targetrefs, thisrefs;double rr, re;- Iterator<String> i = next0.keys(false);+ final Iterator<String> i = next0.keys(false);while (i.hasNext()) {- String targethash = i.next();- String targethost = structure.hostHash2hostName(targethash);+ final String targethash = i.next();+ final String targethost = structure.hostHash2hostName(targethash);if (targethost == null) continue;thisrefs = next.get(targethash).intValue();targetrefs = structure.referencesCount(targethash); // can be cpu/time-critical@@ -177,7 +179,7 @@ public class WebStructurePicture_p {}// recursively set next hostsint nextnodes;- for (String targetHostName: targetHostNames) {+ for (final String targetHostName: targetHostNames) {nextnodes = ((maxnodes <= 0) || (System.currentTimeMillis() >= timeout)) ? 0 : place(graph, structure, targetHostName, bf, maxnodes, timeout, nextlayer, maxlayer, cyc);branches += nextnodes;maxnodes -= nextnodes;diff --git a/htroot/WikiHelp.java b/source/net/yacy/htroot/WikiHelp.javasimilarity index 95%rename from htroot/WikiHelp.javarename to source/net/yacy/htroot/WikiHelp.javaindex 334c99dea..f22e4afa0 100644--- a/htroot/WikiHelp.java+++ b/source/net/yacy/htroot/WikiHelp.java@@ -27,6 +27,8 @@// javac -classpath .:../classes WikiHelp.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;diff --git a/htroot/YaCySearchPluginFF.java b/source/net/yacy/htroot/YaCySearchPluginFF.javasimilarity index 91%rename from htroot/YaCySearchPluginFF.javarename to source/net/yacy/htroot/YaCySearchPluginFF.javaindex 378b235b1..01f2fd8bc 100644--- a/htroot/YaCySearchPluginFF.java+++ b/source/net/yacy/htroot/YaCySearchPluginFF.java@@ -27,6 +27,8 @@//javac -classpath .:../Classes Status.java//if the shell's current path is HTROOT+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.search.Switchboard;import net.yacy.server.serverObjects;@@ -44,8 +46,8 @@ public class YaCySearchPluginFF {final Switchboard sb = (Switchboard) env;final serverObjects prop = new serverObjects();- String host = header.getServerName();- int port = header.getServerPort();+ final String host = header.getServerName();+ final int port = header.getServerPort();prop.put("host", host);prop.put("port", port);diff --git a/htroot/autoconfig.java b/source/net/yacy/htroot/autoconfig.javasimilarity index 92%rename from htroot/autoconfig.javarename to source/net/yacy/htroot/autoconfig.javaindex 3c3598f02..7487f5867 100644--- a/htroot/autoconfig.java+++ b/source/net/yacy/htroot/autoconfig.java@@ -27,6 +27,8 @@//javac -classpath .:../Classes Status.java//if the shell's current path is HTROOT+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.search.SwitchboardConstants;import net.yacy.server.serverObjects;@@ -49,8 +51,8 @@ public class autoconfig {final boolean yacyonly = env.getConfigBool(SwitchboardConstants.PROXY_YACY_ONLY, false);- String host = header.getServerName();- int port = header.getServerPort();+ final String host = header.getServerName();+ final int port = header.getServerPort();prop.put("yacy", yacyonly ? "0" : "1");prop.put("yacy_host", host);diff --git a/htroot/compare_yacy.java b/source/net/yacy/htroot/compare_yacy.javasimilarity index 98%rename from htroot/compare_yacy.javarename to source/net/yacy/htroot/compare_yacy.javaindex 1a2b8d958..4689762e0 100644--- a/htroot/compare_yacy.java+++ b/source/net/yacy/htroot/compare_yacy.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.LinkedHashMap;import java.util.Map;@@ -82,7 +84,7 @@ public class compare_yacy {prop.put("searchengines", searchengines.size());int i = 0;- for (String name: searchengines.keySet()) {+ for (final String name: searchengines.keySet()) {prop.putHTML("searchengines_" + i + "_searchengine", name);prop.put("searchengines_" + i + "_leftengine", name.equals(default_left) ? 1 : 0);prop.put("searchengines_" + i + "_rightengine", name.equals(default_right) ? 1 : 0);diff --git a/htroot/cytag.java b/source/net/yacy/htroot/cytag.javasimilarity index 94%rename from htroot/cytag.javarename to source/net/yacy/htroot/cytag.javaindex 2a629fe2a..e7a11c80c 100644--- a/htroot/cytag.java+++ b/source/net/yacy/htroot/cytag.java@@ -7,7 +7,7 @@// $LastChangedBy$//// LICENSE-//+//// This program is free software; you can redistribute it and/or modify// it under the terms of the GNU General Public License as published by// the Free Software Foundation; either version 2 of the License, or@@ -22,6 +22,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.awt.Image;import java.io.File;import java.io.IOException;@@ -38,14 +40,14 @@ import net.yacy.server.serverSwitch;public class cytag {-+public static Image respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {-+final Switchboard sb = (Switchboard)env;final MultiProtocolURL referer = header.referer();-+// harvest request information- StringBuilder connect = new StringBuilder();+ final StringBuilder connect = new StringBuilder();connect.append('{');appendJSON(connect, "time", GenericFormatter.SHORT_MILSEC_FORMATTER.format());appendJSON(connect, "trail", (referer == null) ? "" : referer.toNormalform(false));@@ -56,11 +58,11 @@ public class cytag {appendJSON(connect, "ip", (ip == null) ? "" : ip);appendJSON(connect, "agent", header.get("User-Agent", ""));connect.append('}');-+if (sb.trail.size() >= 100) sb.trail.remove();sb.trail.add(connect.toString());//Log.logInfo("CYTAG", "catched trail - " + connect.toString());-+final String defaultimage;if (post != null && post.get("icon", "").equals("invisible")) {defaultimage = "invisible.png";@@ -68,7 +70,7 @@ public class cytag {defaultimage = "redpillmini.png";}final File iconfile = new File(sb.getAppPath(), "/htroot/env/grafics/" + defaultimage);-+byte[] imgb = null;try {imgb = FileUtils.read(iconfile);@@ -76,13 +78,13 @@ public class cytag {return null;}if (imgb == null) return null;-+// read imagefinal Image image = ImageParser.parse("cytag.png", imgb);return image;}-+private static final void appendJSON(final StringBuilder sb, final String key, final String value) {if (sb.length() > 2) sb.append(',');sb.append('\"');diff --git a/htroot/goto_p.java b/source/net/yacy/htroot/goto_p.javasimilarity index 93%rename from htroot/goto_p.javarename to source/net/yacy/htroot/goto_p.javaindex 24ccb1d9c..865f04d90 100644--- a/htroot/goto_p.java+++ b/source/net/yacy/htroot/goto_p.java@@ -26,6 +26,8 @@// javac -classpath .:../Classes goto_p.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import java.util.Set;import net.yacy.cora.protocol.RequestHeader;@@ -42,7 +44,7 @@ public class goto_p {/*** url parameter*- * hash= of remote peer+ * hash= of remote peer* path= path part to forward to*/public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {@@ -55,7 +57,7 @@ public class goto_p {}if (hash != null) {- Seed seed = sb.peers.getConnected(hash);+ final Seed seed = sb.peers.getConnected(hash);if (seed != null) {final Set<String> ips = seed.getIPs();diff --git a/htroot/imagetest.java b/source/net/yacy/htroot/imagetest.javasimilarity index 97%rename from htroot/imagetest.javarename to source/net/yacy/htroot/imagetest.javaindex 1cd624645..89f82c75e 100644--- a/htroot/imagetest.java+++ b/source/net/yacy/htroot/imagetest.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;diff --git a/htroot/index.java b/source/net/yacy/htroot/index.javasimilarity index 95%rename from htroot/index.javarename to source/net/yacy/htroot/index.javaindex ce36e34f6..61e4acd97 100644--- a/htroot/index.java+++ b/source/net/yacy/htroot/index.java@@ -28,6 +28,8 @@// javac -classpath .:../classes index.java// if the shell's current path is HTROOT+package net.yacy.htroot;+import net.yacy.cora.document.analysis.Classification;import net.yacy.cora.document.analysis.Classification.ContentDomain;import net.yacy.cora.protocol.RequestHeader;@@ -55,7 +57,7 @@ public class index {// access controlString authenticatedUserName = null;final boolean adminAuthenticated = sb.verifyAuthentication(header);-+if(adminAuthenticated) {authenticatedUserName = sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin");} else {@@ -64,7 +66,7 @@ public class index {authenticatedUserName = user.getUserName();}}- boolean authenticated = adminAuthenticated || authenticatedUserName != null;+ final boolean authenticated = adminAuthenticated || authenticatedUserName != null;if (post != null) {if (post.containsKey("publicPage") && !adminAuthenticated) { // Old style parameter : still in use ?prop.authenticationRequired();@@ -75,16 +77,16 @@ public class index {return prop;}}-+prop.put("authSearch", authenticated);-+boolean global = (post == null) ? true : post.get("resource", "global").equals("global");final boolean focus = (post == null) ? true : post.get("focus", "1").equals("1");int searchoptions = (post == null) ? 0 : Math.min(1, post.getInt("searchoptions", 0));if (!sb.getConfigBool("search.options", true)) {searchoptions = 0;- } else {+ } else {// show heuristic hint on search option screen (only if heuristic is ON by config)prop.put("searchoptions_heuristic", sb.getConfigBool(SwitchboardConstants.HEURISTIC_OPENSEARCH, false));// show date search options (only if dates_in_content_dts search target field is active)@@ -158,9 +160,9 @@ public class index {prop.put("searchdomswitches_searchapp_check", (contentdom == ContentDomain.APP) ? "1" : "0");prop.put("search.navigation", sb.getConfig("search.navigation", "all") );prop.put("search.verify", sb.getConfig("search.verify", "iffresh") );-+handleTopNavBarLoginSection(header, sb, prop, authenticatedUserName);-+// online caution timingsb.localSearchLastAccess = System.currentTimeMillis();@@ -186,15 +188,15 @@ public class index {} else {/* Show a login link */prop.put("showLogin", 2);-+/* The login link targets the same URL as the current location, just adding the 'auth' parameter to indicates that access to extended search features is desired */- StringBuilder loginURL = new StringBuilder("index.html?auth");+ final StringBuilder loginURL = new StringBuilder("index.html?auth");final String query = header.getQueryString();if(query != null) {loginURL.append("&").append(query);}--++prop.put("showLogin_loginURL", loginURL.toString());}} else {diff --git a/htroot/jslicense.java b/source/net/yacy/htroot/jslicense.javasimilarity index 95%rename from htroot/jslicense.javarename to source/net/yacy/htroot/jslicense.javaindex 27c1e3c54..45d3115a0 100755--- a/htroot/jslicense.java+++ b/source/net/yacy/htroot/jslicense.java@@ -27,6 +27,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;diff --git a/htroot/mediawiki_p.java b/source/net/yacy/htroot/mediawiki_p.javasimilarity index 99%rename from htroot/mediawiki_p.javarename to source/net/yacy/htroot/mediawiki_p.javaindex 51728ef30..69b9fb0db 100644--- a/htroot/mediawiki_p.java+++ b/source/net/yacy/htroot/mediawiki_p.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.File;import java.io.IOException;diff --git a/htroot/opensearchdescription.java b/source/net/yacy/htroot/opensearchdescription.javasimilarity index 98%rename from htroot/opensearchdescription.javarename to source/net/yacy/htroot/opensearchdescription.javaindex 8364fea59..03fd790c1 100644--- a/htroot/opensearchdescription.java+++ b/source/net/yacy/htroot/opensearchdescription.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.http.servlets.YaCyDefaultServlet;import net.yacy.search.Switchboard;@@ -40,7 +42,7 @@ public class opensearchdescription {if (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");final String thisContext = YaCyDefaultServlet.getContext(header, sb);-+final serverObjects prop = new serverObjects();prop.put("compareyacy", post != null && post.getBoolean("compare_yacy") ? 1 : 0);prop.putXML("thisContext", thisContext);diff --git a/htroot/osm.java b/source/net/yacy/htroot/osm.javasimilarity index 94%rename from htroot/osm.javarename to source/net/yacy/htroot/osm.javaindex 09828d3bb..1671e8957 100644--- a/htroot/osm.java+++ b/source/net/yacy/htroot/osm.java@@ -7,17 +7,19 @@* modify it under the terms of the GNU Lesser General Public* License as published by the Free Software Foundation; either* version 2.1 of the License, or (at your option) any later version.- *+ ** This library is distributed in the hope that it will be useful,* but WITHOUT ANY WARRANTY; without even the implied warranty of* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU* Lesser General Public License for more details.- *+ ** You should have received a copy of the GNU Lesser General Public License* along with this program in the file lgpl21.txt* If not, see <http://www.gnu.org/licenses/>.*/+package net.yacy.htroot;+import net.yacy.cora.protocol.HeaderFramework;import net.yacy.cora.protocol.RequestHeader;import net.yacy.peers.graphics.EncodedImage;@@ -47,13 +49,13 @@ public class osm {}final OSMTile.tileCoordinates coord = new OSMTile.tileCoordinates(lat, lon, zoom);- RasterPlotter map = OSMTile.getCombinedTiles(coord, width, height);+ final RasterPlotter map = OSMTile.getCombinedTiles(coord, width, height);map.setDrawMode(DrawMode.MODE_SUB);map.setColor(0xffffff);/** copyright notice on OSM Tiles* According to http://www.openstreetmap.org/copyright/ the (C) of the map tiles is (CC BY-SA)- * while the OpenStreetMap raw data is licensed with (ODbL) http://opendatacommons.org/licenses/odbl/+ * while the OpenStreetMap raw data is licensed with (ODbL) http://opendatacommons.org/licenses/odbl/* Map tiles shall be underlined with the statement "(C) OpenStreetMap contributors". In our 5-dot character* set the lowercase letters do not look good, so we use uppercase only.* The (C) symbol is not available in our font, so we use the letters (C) instead.diff --git a/htroot/rct_p.java b/source/net/yacy/htroot/rct_p.javasimilarity index 99%rename from htroot/rct_p.javarename to source/net/yacy/htroot/rct_p.javaindex a27e3e27a..7c7bf5384 100644--- a/htroot/rct_p.java+++ b/source/net/yacy/htroot/rct_p.java@@ -25,6 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.net.MalformedURLException;import java.util.Date;import java.util.Iterator;diff --git a/htroot/robots.java b/source/net/yacy/htroot/robots.javasimilarity index 99%rename from htroot/robots.javarename to source/net/yacy/htroot/robots.javaindex df87cf5e7..06b82ffd3 100644--- a/htroot/robots.java+++ b/source/net/yacy/htroot/robots.java@@ -4,6 +4,8 @@//$LastChangedBy$//+package net.yacy.htroot;+import java.io.File;import java.util.ArrayList;import java.util.List;@@ -58,7 +60,7 @@ public class robots {}}}-+if (rbc.isLockedDisallowed()) {prop.put(RobotsTxtConfig.ALL + "_" + RobotsTxtConfig.LOCKED, htrootFiles.size());for (int i = 0; i < htrootFiles.size(); i++) {diff --git a/htroot/sharedBlacklist_p.java b/source/net/yacy/htroot/sharedBlacklist_p.javasimilarity index 96%rename from htroot/sharedBlacklist_p.javarename to source/net/yacy/htroot/sharedBlacklist_p.javaindex c22059c12..18eb528b0 100644--- a/htroot/sharedBlacklist_p.java+++ b/source/net/yacy/htroot/sharedBlacklist_p.java@@ -29,6 +29,8 @@//javac -classpath .:../Classes Blacklist_p.java//if the shell's current path is HTROOT+package net.yacy.htroot;+import java.io.File;import java.io.IOException;import java.io.StringReader;@@ -53,8 +55,8 @@ import net.yacy.document.parser.html.CharacterCoding;import net.yacy.kelondro.util.FileUtils;import net.yacy.peers.Seed;import net.yacy.repository.Blacklist;-import net.yacy.repository.BlacklistHostAndPath;import net.yacy.repository.Blacklist.BlacklistType;+import net.yacy.repository.BlacklistHostAndPath;import net.yacy.search.Switchboard;import net.yacy.search.SwitchboardConstants;import net.yacy.search.query.SearchEventCache;@@ -131,7 +133,7 @@ public class sharedBlacklist_p {Iterator<String> otherBlacklist = null;ListAccumulator otherBlacklists = null;- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));if (post.containsKey("hash")) {/* ======================================================@@ -152,11 +154,11 @@ public class sharedBlacklist_p {final String Port = seed.get(Seed.PORT, "8090");final String peerName = seed.get(Seed.NAME, "<" + IP + ":" + Port + ">");prop.putHTML("page_source", peerName);-+downloadURLOld = seed.getPublicURL(IP,sb.getConfigBool(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT)) + "/yacy/list.html?col=black";-+// download the blacklisttry {// get Listdiff --git a/htroot/ssitestservlet.java b/source/net/yacy/htroot/ssitestservlet.javasimilarity index 98%rename from htroot/ssitestservlet.javarename to source/net/yacy/htroot/ssitestservlet.javaindex 274c57b41..fef629f25 100644--- a/htroot/ssitestservlet.java+++ b/source/net/yacy/htroot/ssitestservlet.java@@ -25,6 +25,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;diff --git a/htroot/suggest.java b/source/net/yacy/htroot/suggest.javasimilarity index 96%rename from htroot/suggest.javarename to source/net/yacy/htroot/suggest.javaindex 609a93ff0..c82c0e1ce 100644--- a/htroot/suggest.java+++ b/source/net/yacy/htroot/suggest.java@@ -22,6 +22,8 @@// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Collection;import java.util.ConcurrentModificationException;@@ -59,7 +61,7 @@ public class suggest {final String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, "");final boolean json = ext.equals("json");final boolean xml = ext.equals("xml");-+// get queryfinal String originalquerystring = (post == null) ? "" : post.get("query", post.get("q", ""));final String querystring = originalquerystring.trim();@@ -71,10 +73,10 @@ public class suggest {final Collection<StringBuilder> suggestions = didYouMean.getSuggestions(timeout, count, sb.index.fulltext().collectionSize() < 2000000);//[#[query]#,[#{suggestions}##[text]##(eol)#,::#(/eol)##{/suggestions}#]]synchronized (suggestions) {- for (StringBuilder suggestion: suggestions) {+ for (final StringBuilder suggestion: suggestions) {if (c >= meanMax) break;try {- String s = suggestion.toString();+ final String s = suggestion.toString();if (json) {prop.putJSON("suggestions_" + c + "_text", s);} else if (xml) {diff --git a/htroot/test.java b/source/net/yacy/htroot/test.javasimilarity index 96%rename from htroot/test.javarename to source/net/yacy/htroot/test.javaindex baa3efb01..70601bbb5 100644--- a/htroot/test.java+++ b/source/net/yacy/htroot/test.java@@ -4,6 +4,8 @@//$LastChangedBy$//+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;diff --git a/htroot/yacyinteractive.java b/source/net/yacy/htroot/yacyinteractive.javasimilarity index 94%rename from htroot/yacyinteractive.javarename to source/net/yacy/htroot/yacyinteractive.javaindex 9a91763b9..644cfbfa0 100644--- a/htroot/yacyinteractive.java+++ b/source/net/yacy/htroot/yacyinteractive.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.search.Switchboard;import net.yacy.search.SwitchboardConstants;@@ -36,8 +38,8 @@ public class yacyinteractive {private static long indeSizeCache = 0;private static long indexSizeTime = 0;-- public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, serverObjects post, final serverSwitch env) {++ public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {final Switchboard sb = (Switchboard) env;final serverObjects prop = new serverObjects();@@ -59,7 +61,7 @@ public class yacyinteractive {prop.put("serverlist", query.isEmpty() ? 1 : 0);prop.put("focus", focus ? 1 : 0);- long t = System.currentTimeMillis();+ final long t = System.currentTimeMillis();if (t - indexSizeTime > 60000) {indeSizeCache = sb.index.fulltext().collectionSize();indexSizeTime = t;diff --git a/htroot/yacysearch.java b/source/net/yacy/htroot/yacysearch.javasimilarity index 97%rename from htroot/yacysearch.javarename to source/net/yacy/htroot/yacysearch.javaindex c318f7baf..ac728a4a3 100644--- a/htroot/yacysearch.java+++ b/source/net/yacy/htroot/yacysearch.java@@ -27,6 +27,10 @@// javac -classpath .:../classes yacysearch.java// if the shell's current path is HTROOT+package net.yacy.htroot;++import static net.yacy.repository.BlacklistHelper.addBlacklistEntry;+import java.io.IOException;import java.io.UnsupportedEncodingException;import java.net.IDN;@@ -80,7 +84,6 @@ import net.yacy.peers.EventChannel;import net.yacy.peers.NewsPool;import net.yacy.peers.graphics.ProfilingGraph;import net.yacy.repository.Blacklist;-import static net.yacy.repository.BlacklistHelper.addBlacklistEntry;import net.yacy.search.EventTracker;import net.yacy.search.SearchAccessRateConstants;import net.yacy.search.Switchboard;@@ -149,16 +152,16 @@ public class yacysearch {prop.put("authSearch", authenticatedUserName != null);// produce vocabulary navigation sidebars- Collection<Tagging> vocabularies = LibraryProvider.autotagging.getVocabularies();+ final Collection<Tagging> vocabularies = LibraryProvider.autotagging.getVocabularies();int j = 0;- for (Tagging v: vocabularies) {+ for (final Tagging v: vocabularies) {prop.put("sidebarVocabulary_" + j + "_vocabulary", v.getName());j++;}prop.put("sidebarVocabulary", j);// get segment- Segment indexSegment = sb.index;+ final Segment indexSegment = sb.index;final String EXT = header.get(HeaderFramework.CONNECTION_PROP_EXT, "");final boolean rss = "rss.atom".contains(EXT);@@ -166,15 +169,15 @@ public class yacysearch {prop.put("promoteSearchPageGreeting", promoteSearchPageGreeting);// adding some additional properties needed for the rss feed- String peerContext = YaCyDefaultServlet.getContext(header, sb);+ final String peerContext = YaCyDefaultServlet.getContext(header, sb);prop.put("searchBaseURL", peerContext + "/yacysearch.html");prop.put("rssYacyImageURL", peerContext + "/env/grafics/yacy.png");prop.put("thisaddress", peerContext);final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER);final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, true) || clustersearch;- boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;+ final boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode);- boolean stealthmode = p2pmode && !global;+ final boolean stealthmode = p2pmode && !global;if ( post == null || indexSegment == null || env == null || !searchAllowed ) {if (indexSegment == null) ConcurrentLog.info("yacysearch", "indexSegment == null");@@ -247,7 +250,7 @@ public class yacysearch {}// time zone- int timezoneOffset = post.getInt("timezoneOffset", 0);+ final int timezoneOffset = post.getInt("timezoneOffset", 0);// collect search attributes@@ -479,7 +482,7 @@ public class yacysearch {// read collectionmodifier.collection = post.get("collection", modifier.collection); // post arguments may overrule parsed collection values- int stp = querystring.indexOf('*');+ final int stp = querystring.indexOf('*');if (stp >= 0) {// if the star appears as a single entry, use the catchallstringif (querystring.length() == 1) {@@ -536,10 +539,10 @@ public class yacysearch {}int voc = 0;- Collection<Tagging.Metatag> metatags = new ArrayList<Tagging.Metatag>(1);+ final Collection<Tagging.Metatag> metatags = new ArrayList<Tagging.Metatag>(1);while ((voc = querystring.indexOf("/vocabulary/", 0)) >= 0) {String vocabulary = "";- int ve = querystring.indexOf(' ', voc + 12);+ final int ve = querystring.indexOf(' ', voc + 12);if (ve < 0) {vocabulary = querystring.substring(voc);querystring = querystring.substring(0, voc).trim();@@ -549,11 +552,11 @@ public class yacysearch {}modifier.add(vocabulary);vocabulary = vocabulary.substring(12);- int p = vocabulary.indexOf('/');+ final int p = vocabulary.indexOf('/');if (p > 0) {- String k = vocabulary.substring(0, p);- String v = vocabulary.substring(p + 1);- Metatag mt = LibraryProvider.autotagging.metatag(k, v);+ final String k = vocabulary.substring(0, p);+ final String v = vocabulary.substring(p + 1);+ final Metatag mt = LibraryProvider.autotagging.metatag(k, v);if (mt != null) {metatags.add(mt);} else {@@ -565,7 +568,7 @@ public class yacysearch {int radius = 0;double lon = 0.0d, lat = 0.0d, rad = 0.0d;if ((radius = querystring.indexOf("/radius/")) >= 0) {- int ve = querystring.indexOf(' ', radius + 8);+ final int ve = querystring.indexOf(' ', radius + 8);String geo = "";if (ve < 0) {geo = querystring.substring(radius);@@ -575,7 +578,7 @@ public class yacysearch {querystring = querystring.substring(0, radius) + querystring.substring(ve);}geo = geo.substring(8);- String[] sp = geo.split("/");+ final String[] sp = geo.split("/");if (sp.length == 3) try {lat = Double.parseDouble(sp[0]);lon = Double.parseDouble(sp[1]);@@ -717,7 +720,7 @@ public class yacysearch {bmk.setProperty(Bookmark.BOOKMARK_QUERY, querystring);bmk.addTag("/search"); // add to bookmark folderbmk.addTag("searchresult"); // add tag- String urlhash = post.get("bookmarkref");+ final String urlhash = post.get("bookmarkref");final URIMetadataNode urlentry = indexSegment.fulltext().getMetadata(UTF8.getBytes(urlhash));if (urlentry != null && !urlentry.dc_title().isEmpty()) {bmk.setProperty(Bookmark.BOOKMARK_TITLE, urlentry.dc_title());@@ -735,13 +738,13 @@ public class yacysearch {return prop;}- String blacklisturl = crypt.simpleDecode(post.get("blacklisturl", "")); // url+ final String blacklisturl = crypt.simpleDecode(post.get("blacklisturl", "")); // urltry {- MultiProtocolURL mpurl = new MultiProtocolURL(blacklisturl);+ final MultiProtocolURL mpurl = new MultiProtocolURL(blacklisturl);addBlacklistEntry(Blacklist.defaultBlacklist(sb.listsPath),mpurl.getHost() + "/.*");- } catch (MalformedURLException e) {+ } catch (final MalformedURLException e) {}}@@ -1037,11 +1040,11 @@ public class yacysearch {prop.put("focus", focus ? 1 : 0); // focus search fieldprop.put("searchagain", global ? "1" : "0");- String former = originalquerystring.replaceAll(Segment.catchallString, "*"); // hide catchallString in output+ final String former = originalquerystring.replaceAll(Segment.catchallString, "*"); // hide catchallString in outputprop.putHTML("former", former);try {prop.put("formerEncoded", URLEncoder.encode(former, StandardCharsets.UTF_8.name()));- } catch (UnsupportedEncodingException e) {+ } catch (final UnsupportedEncodingException e) {ConcurrentLog.warn("LOCAL_SEARCH", "Unsupported UTF-8 encoding!");prop.put("formerEncoded", former);}diff --git a/htroot/yacysearch_location.java b/source/net/yacy/htroot/yacysearch_location.javasimilarity index 97%rename from htroot/yacysearch_location.javarename to source/net/yacy/htroot/yacysearch_location.javaindex 865cca097..3c119986f 100644--- a/htroot/yacysearch_location.java+++ b/source/net/yacy/htroot/yacysearch_location.java@@ -17,6 +17,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.util.Date;import java.util.Set;import java.util.concurrent.BlockingQueue;@@ -44,7 +46,7 @@ public class yacysearch_location {final serverObjects prop = new serverObjects();prop.put("kml", 0);-+final String peerContext = YaCyDefaultServlet.getContext(header, sb);if (header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("kml") ||@@ -68,7 +70,7 @@ public class yacysearch_location {final double lat = post.getDouble("lat", 0.0d);final double radius = post.getDouble("r", 0.0d);//i.e. http://localhost:8090/yacysearch_location.kml?query=berlin&maximumTime=2000&maximumRecords=100-+int placemarkCounter = 0;if (query.length() > 0 && search_query) {final Set<GeoLocation> locations = LibraryProvider.geoLoc.find(query, true);@@ -77,8 +79,8 @@ public class yacysearch_location {}for (final GeoLocation location: locations) {// write for all locations a point to this message- double lo = location.lon();- double la = location.lat();+ final double lo = location.lon();+ final double la = location.lat();if (lo == 0.0d && la == 0.0d) continue; // bad locationprop.put("kml_placemark_" + placemarkCounter + "_location", location.getName());prop.put("kml_placemark_" + placemarkCounter + "_name", location.getName());@@ -105,8 +107,8 @@ public class yacysearch_location {RSSMessage message;loop: while ((message = results.poll(maximumTime, TimeUnit.MILLISECONDS)) != RSSMessage.POISON) {if (message == null) break loop;- double lo = message.getLon();- double la = message.getLat();+ final double lo = message.getLon();+ final double la = message.getLat();if (lo == 0.0d && la == 0.0d) continue; // bad locationprop.put("kml_placemark_" + placemarkCounter + "_location", message.getTitle());prop.put("kml_placemark_" + placemarkCounter + "_name", message.getTitle());diff --git a/htroot/yacysearchitem.java b/source/net/yacy/htroot/yacysearchitem.javasimilarity index 94%rename from htroot/yacysearchitem.javarename to source/net/yacy/htroot/yacysearchitem.javaindex a83d478f9..fafa3959d 100644--- a/htroot/yacysearchitem.java+++ b/source/net/yacy/htroot/yacysearchitem.java@@ -24,6 +24,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.awt.Dimension;import java.io.File;import java.io.UnsupportedEncodingException;@@ -65,7 +67,6 @@ import net.yacy.kelondro.util.Formatter;import net.yacy.peers.NewsPool;import net.yacy.peers.Seed;import net.yacy.peers.graphics.ProfilingGraph;-import net.yacy.repository.Blacklist;import net.yacy.search.EventTracker;import net.yacy.search.Switchboard;import net.yacy.search.SwitchboardConstants;@@ -160,7 +161,7 @@ public class yacysearchitem {final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "");final boolean noreferrer = sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_NOREFERRER, SwitchboardConstants.SEARCH_RESULT_NOREFERRER_DEFAULT);- long timeout = item == 0 ? 10000 : (theSearch.query.isLocal() ? 1000 : 3000);+ final long timeout = item == 0 ? 10000 : (theSearch.query.isLocal() ? 1000 : 3000);if (theSearch.query.contentdom == Classification.ContentDomain.TEXT || theSearch.query.contentdom == Classification.ContentDomain.ALL) {// text search@@ -234,8 +235,8 @@ public class yacysearchitem {// END interaction- boolean isAtomFeed = header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("atom");- String resultFileName = resultURL.getFileName();+ final boolean isAtomFeed = header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("atom");+ final String resultFileName = resultURL.getFileName();prop.putHTML("content_target", target);DigestURL faviconURL = null;final boolean showFavicon = sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON,@@ -264,7 +265,7 @@ public class yacysearchitem {try {prop.putXML("content_image_url", result.imageURL());prop.put("content_image", 1);- } catch (UnsupportedOperationException e) {+ } catch (final UnsupportedOperationException e) {/* May occur when the document embedded images information is incomplete to retrieve at least an valid image url*/prop.put("content_image", 0);@@ -273,12 +274,12 @@ public class yacysearchitem {prop.put("content_urlhash", urlhash);prop.put("content_ranking", Float.toString(result.score()));- Date[] events = result.events();- boolean showEvent = events != null && events.length > 0 && sb.getConfig("search.navigation", "").indexOf("date",0) >= 0;+ final Date[] events = result.events();+ final boolean showEvent = events != null && events.length > 0 && sb.getConfig("search.navigation", "").indexOf("date",0) >= 0;prop.put("content_showEvent", showEvent ? 1 : 0);- Collection<File> snapshotPaths = sb.getConfigBool("search.result.show.snapshots", true) ? Transactions.findPaths(result.url(), null, State.ANY) : null;+ final Collection<File> snapshotPaths = sb.getConfigBool("search.result.show.snapshots", true) ? Transactions.findPaths(result.url(), null, State.ANY) : null;if (fileType == FileType.HTML) { // html template specific settings- boolean showKeywords = (sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS,+ final boolean showKeywords = (sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS,SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS_DEFAULT) && !result.dc_subject().isEmpty());prop.put("content_showKeywords", showKeywords);prop.put("content_showDate", sb.getConfigBool("search.result.show.date", true) && !showEvent ? 1 : 0);@@ -298,8 +299,8 @@ public class yacysearchitem {if (showKeywords) { // tokenize keywordsfinal StringTokenizer stoc = new StringTokenizer(result.dc_subject()," ");String rawNavQueryModifier;- Navigator navi = theSearch.navigatorPlugins.get("keywords");- boolean naviAvail = navi != null;+ final Navigator navi = theSearch.navigatorPlugins.get("keywords");+ final boolean naviAvail = navi != null;final int firstMaxKeywords = sb.getConfigInt(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT,SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT_DEFAULT);int i = 0;@@ -347,9 +348,9 @@ public class yacysearchitem {prop.put("content_showIndexBrowser_link", resultUrlstring);if (sb.getConfigBool("search.result.show.vocabulary", true)) {int c = 0;- for (String key: result.getFieldNames()) {+ for (final String key: result.getFieldNames()) {if (key.startsWith("vocabulary_") && key.endsWith("_sxt")) {- Collection<Object> terms = result.getFieldValues(key);+ final Collection<Object> terms = result.getFieldValues(key);prop.putHTML("content_showVocabulary_vocabulary_" + c + "_name", key.substring(11, key.length() - 4));prop.putHTML("content_showVocabulary_vocabulary_" + c + "_terms", terms.toString());c++;@@ -447,8 +448,8 @@ public class yacysearchitem {final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER);final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, true) || clustersearch;- boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;- boolean stealthmode = p2pmode && theSearch.query.isLocal();+ final boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;+ final boolean stealthmode = p2pmode && theSearch.query.isLocal();if ((sb.getConfigBool(SwitchboardConstants.HEURISTIC_SEARCHRESULTS, false) ||(sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ACTIVE, false) && sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ENABLED, false) && Memory.getSystemLoadAverage() < 1.0)) &&!stealthmode) sb.heuristicSearchResults(result);@@ -631,16 +632,16 @@ public class yacysearchitem {* @return favicon URL or null when even default favicon URL can not be generated* @throws NullPointerException when one requested parameter is null*/- protected static DigestURL getFaviconURL(final URIMetadataNode result, Dimension preferredSize) {+ protected static DigestURL getFaviconURL(final URIMetadataNode result, final Dimension preferredSize) {/** We look preferably for a standard icon with preferred size, but* accept as a fallback other icons below 128x128 or with no known size*/- IconEntry faviconEntry = result.getFavicon(preferredSize);+ final IconEntry faviconEntry = result.getFavicon(preferredSize);DigestURL faviconURL;if (faviconEntry == null) {try {- String defaultFaviconURL = result.url().getProtocol() + "://" + result.url().getHost()+ final String defaultFaviconURL = result.url().getProtocol() + "://" + result.url().getHost()+ ((result.url().getPort() != -1) ? (":" + result.url().getPort()) : "") + "/favicon.ico";faviconURL = new DigestURL(defaultFaviconURL);} catch (final MalformedURLException e1) {@@ -662,9 +663,9 @@ public class yacysearchitem {* @return url to propose in search result or empty string when faviconURL* is null*/- private static String processFaviconURL(final boolean hasFullViewingRights, DigestURL faviconURL) {+ private static String processFaviconURL(final boolean hasFullViewingRights, final DigestURL faviconURL) {/* Only use licence code for non authentified users. For authenticated users licence would never be released and would unnecessarily fill URLLicense.permissions. */- StringBuilder contentFaviconURL = new StringBuilder();+ final StringBuilder contentFaviconURL = new StringBuilder();if (faviconURL != null) {final String iconUrlExt = MultiProtocolURL.getFileExtension(faviconURL.getFileName());/* Image format ouput for ViewFavicon servlet : default is png, except with gif and svg icons */@@ -695,9 +696,9 @@ public class yacysearchitem {*/private static void addAuthorizedActions(final Switchboard sb, final serverObjects prop,final SearchEvent theSearch, final String resultUrlstring, final String resource, final String origQ,- final String urlhash, UserDB.Entry user) {+ final String urlhash, final UserDB.Entry user) {// check if url exists in bookmarks- boolean bookmarkexists = sb.bookmarksDB.getBookmark(urlhash) != null;+ final boolean bookmarkexists = sb.bookmarksDB.getBookmark(urlhash) != null;if (user == null || user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT)) {prop.put("content_authorized_bookmark", !bookmarkexists);} else@@ -715,17 +716,17 @@ public class yacysearchitem {String encodedURLString;try {encodedURLString = URLEncoder.encode(crypt.simpleEncode(resultUrlstring), StandardCharsets.UTF_8.name());- } catch (UnsupportedEncodingException e1) {+ } catch (final UnsupportedEncodingException e1) {ConcurrentLog.warn("YACY_SEARCH_ITEM", "UTF-8 encoding is not supported!");encodedURLString = crypt.simpleEncode(resultUrlstring);}final String bookmarkLink = linkBuilder.append("&bookmarkurl=").append(encodedURLString).append("&bookmarkref=" + urlhash).toString();linkBuilder.setLength(baseUrlLength);- String deleteLink = linkBuilder.append("&deleteref=").append(urlhash).toString();+ final String deleteLink = linkBuilder.append("&deleteref=").append(urlhash).toString();linkBuilder.setLength(baseUrlLength);- String recommendLink = linkBuilder.append("&recommendref=").append(urlhash).toString();+ final String recommendLink = linkBuilder.append("&recommendref=").append(urlhash).toString();linkBuilder.setLength(baseUrlLength);final String blacklistLink = linkBuilder.append("&blacklisturl=").append(encodedURLString).toString();@@ -760,10 +761,10 @@ public class yacysearchitem {* @param noreferrer set to true when the noreferrer link type should be added to the original image source links*/private static void processImage(final Switchboard sb, final serverObjects prop, final int item,- final SearchEvent theSearch, final String target_special_pattern, long timeout, boolean fullViewingRights, final boolean noreferrer) {+ final SearchEvent theSearch, final String target_special_pattern, final long timeout, final boolean fullViewingRights, final boolean noreferrer) {prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific contenttry {- SearchEvent.ImageResult image = theSearch.oneImageResult(item, timeout, theSearch.query.isStrictContentDom());+ final SearchEvent.ImageResult image = theSearch.oneImageResult(item, timeout, theSearch.query.isStrictContentDom());final String imageUrlstring = image.imageUrl.toNormalform(true);final String imageUrlExt = MultiProtocolURL.getFileExtension(image.imageUrl.getFileName());final String target = sb.getConfig(imageUrlstring.matches(target_special_pattern) ? SwitchboardConstants.SEARCH_TARGET_SPECIAL : SwitchboardConstants.SEARCH_TARGET_DEFAULT, "_self");@@ -772,7 +773,7 @@ public class yacysearchitem {/* Image format ouput for ViewImage servlet : default is png, except with gif and svg images */final String viewImageExt = !imageUrlExt.isEmpty() && ImageViewer.isBrowserRendered(imageUrlExt) ? imageUrlExt : "png";/* Thumb URL */- StringBuilder thumbURLBuilder = new StringBuilder("ViewImage.").append(viewImageExt).append("?maxwidth=")+ final StringBuilder thumbURLBuilder = new StringBuilder("ViewImage.").append(viewImageExt).append("?maxwidth=").append(DEFAULT_IMG_WIDTH).append("&maxheight=").append(DEFAULT_IMG_HEIGHT).append("&isStatic=true&quadratic");/* Only use licence code for non authentified users. For authenticated users licence would never be released and would unnecessarily fill URLLicense.permissions. */@@ -781,7 +782,7 @@ public class yacysearchitem {} else {thumbURLBuilder.append("&code=").append(URLLicense.aquireLicense(image.imageUrl));}- String thumbURL = thumbURLBuilder.toString();+ final String thumbURL = thumbURLBuilder.toString();prop.putHTML("content_item_hrefCache", thumbURL);/* Full size preview URL */if(fullViewingRights) {@@ -807,8 +808,8 @@ public class yacysearchitem {itemWidth = "";itemHeight = "100%";if(image.height > 0) {- double scale = ((double)DEFAULT_IMG_HEIGHT) / ((double)image.height);- int margin = (int)((image.height - image.width) * (scale / 2.0));+ final double scale = ((double)DEFAULT_IMG_HEIGHT) / ((double)image.height);+ final int margin = (int)((image.height - image.width) * (scale / 2.0));itemStyle = "margin-left: " + margin + "px;";}} else {@@ -816,8 +817,8 @@ public class yacysearchitem {itemWidth = "100%";itemHeight = "";if(image.height > image.width && image.width > 0) {- double scale = ((double)DEFAULT_IMG_WIDTH) / ((double)image.width);- int margin = (int)((image.width - image.height) * (scale / 2.0));+ final double scale = ((double)DEFAULT_IMG_WIDTH) / ((double)image.width);+ final int margin = (int)((image.width - image.height) * (scale / 2.0));itemStyle = "margin-top: " + margin + "px;";}}@@ -833,7 +834,7 @@ public class yacysearchitem {prop.put("content_item_sourcedom", image.sourceUrl.getHost());prop.put("content_item_nl", (item == theSearch.query.offset) ? 0 : 1);prop.put("content_item", 1);- } catch (MalformedURLException e) {+ } catch (final MalformedURLException e) {prop.put("content_item", "0");}}diff --git a/htroot/yacysearchlatestinfo.java b/source/net/yacy/htroot/yacysearchlatestinfo.javasimilarity index 97%rename from htroot/yacysearchlatestinfo.javarename to source/net/yacy/htroot/yacysearchlatestinfo.javaindex 321fe466c..5a77b0fd1 100644--- a/htroot/yacysearchlatestinfo.java+++ b/source/net/yacy/htroot/yacysearchlatestinfo.java@@ -1,3 +1,5 @@+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.data.UserDB;import net.yacy.http.servlets.TemplateMissingParameterException;@@ -16,10 +18,10 @@ public class yacysearchlatestinfo {if (post == null) {throw new TemplateMissingParameterException("The eventID parameter is required");}-+final serverObjects prop = new serverObjects();- Switchboard sb = (Switchboard) env;-+ final Switchboard sb = (Switchboard) env;+final boolean adminAuthenticated = sb.verifyAuthentication(header);final UserDB.Entry user = sb.userDB != null ? sb.userDB.getUser(header) : null;final boolean userAuthenticated = (user != null && user.hasRight(UserDB.AccessRight.EXTENDED_SEARCH_RIGHT));diff --git a/htroot/yacysearchpagination.java b/source/net/yacy/htroot/yacysearchpagination.javasimilarity index 97%rename from htroot/yacysearchpagination.javarename to source/net/yacy/htroot/yacysearchpagination.javaindex 1a41b5f61..4556b216f 100644--- a/htroot/yacysearchpagination.java+++ b/source/net/yacy/htroot/yacysearchpagination.java@@ -21,6 +21,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import net.yacy.cora.protocol.RequestHeader;import net.yacy.http.servlets.TemplateMissingParameterException;import net.yacy.search.Switchboard;@@ -61,7 +63,7 @@ public class yacysearchpagination {final int defaultItemsPerPage = sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10);/* Detailed rules on items per page limits are handle in yacysearch.html */- int itemsPerPage = Math.max(1, post.getInt("maximumRecords", defaultItemsPerPage));+ final int itemsPerPage = Math.max(1, post.getInt("maximumRecords", defaultItemsPerPage));final SearchEvent theSearch = SearchEventCache.getEvent(eventID);if (theSearch == null) {@@ -71,9 +73,9 @@ public class yacysearchpagination {prop.put("pagination", false);} else {prop.put("pagination", true);-+final RequestHeader.FileType fileType = header.fileType();-+if(jsResort) {/* Pagination links are processed on browser side : just prepare prev and next buttons */prop.put("pagination_hidePagination", true);@@ -125,7 +127,7 @@ public class yacysearchpagination {.navurl(fileType, activePage + 1, theSearch.query, null, false, authFeatures).toString());}}-+}diff --git a/htroot/yacysearchtrailer.java b/source/net/yacy/htroot/yacysearchtrailer.javasimilarity index 92%rename from htroot/yacysearchtrailer.javarename to source/net/yacy/htroot/yacysearchtrailer.javaindex c32fdaecb..3b9e7a266 100644--- a/htroot/yacysearchtrailer.java+++ b/source/net/yacy/htroot/yacysearchtrailer.java@@ -19,6 +19,8 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA+package net.yacy.htroot;+import java.io.UnsupportedEncodingException;import java.net.URLEncoder;import java.nio.charset.StandardCharsets;@@ -81,7 +83,7 @@ public class yacysearchtrailer {/** Authenticated search is explicitely requested here* but no authentication is provided : ask now for authentication.- * Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information+ * Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information* and as this page is not private, protected features would simply be hidden without asking browser again for authentication.* (see mantis 766 : http://mantis.tokeek.de/view.php?id=766) **/@@ -99,12 +101,12 @@ public class yacysearchtrailer {final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER);final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, true) || clustersearch;- boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;- boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode);- boolean stealthmode = p2pmode && !global;+ final boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;+ final boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode);+ final boolean stealthmode = p2pmode && !global;// compose search navigation- ContentDomain contentdom = theSearch.getQuery().contentdom;+ final ContentDomain contentdom = theSearch.getQuery().contentdom;prop.put("searchdomswitches",sb.getConfigBool("search.text", true)|| sb.getConfigBool("search.audio", true)@@ -169,22 +171,22 @@ public class yacysearchtrailer {navigatorIterator = topicNavigator.keys(false);int i = 0;// first sort the list to a form where the greatest element is in the middle- LinkedList<Map.Entry<String, Integer>> cloud = new LinkedList<Map.Entry<String, Integer>>();+ final LinkedList<Map.Entry<String, Integer>> cloud = new LinkedList<Map.Entry<String, Integer>>();int mincount = Integer.MAX_VALUE; int maxcount = 0;while (i < TOPWORDS_MAXCOUNT && navigatorIterator.hasNext()) {name = navigatorIterator.next();count = topicNavigator.get(name);if (count == 0) break;if (name == null) continue;- int normcount = (count + TOPWORDS_MAXCOUNT - i) / 2;+ final int normcount = (count + TOPWORDS_MAXCOUNT - i) / 2;if (normcount > maxcount) maxcount = normcount;if (normcount < mincount) mincount = normcount;- Map.Entry<String, Integer> entry = new AbstractMap.SimpleEntry<String, Integer>(name, normcount);+ final Map.Entry<String, Integer> entry = new AbstractMap.SimpleEntry<String, Integer>(name, normcount);if (cloud.size() % 2 == 0) cloud.addFirst(entry); else cloud.addLast(entry); // alternating add entry to first or last position.i++;}i= 0;- for (Map.Entry<String, Integer> entry: cloud) {+ for (final Map.Entry<String, Integer> entry: cloud) {name = entry.getKey();count = entry.getValue();prop.put(fileType, "nav-topics_element_" + i + "_modifier", name);@@ -211,12 +213,12 @@ public class yacysearchtrailer {prop.put("nav-protocols", 1);//int httpCount = theSearch.protocolNavigator.delete("http");//int httpsCount = theSearch.protocolNavigator.delete("https");- //theSearch.protocolNavigator.inc("http(s)", httpCount + httpsCount);+ //theSearch.protocolNavigator.inc("http(s)", httpCount + httpsCount);navigatorIterator = theSearch.protocolNavigator.keys(false);int i = 0, pos = 0, neg = 0;String nav, rawNav;- String oldQuery = theSearch.query.getQueryGoal().query_original; // prepare hack to make radio-button like navigation- String oldProtocolModifier = theSearch.query.modifier.protocol;+ final String oldQuery = theSearch.query.getQueryGoal().query_original; // prepare hack to make radio-button like navigation+ final String oldProtocolModifier = theSearch.query.modifier.protocol;if (oldProtocolModifier != null && oldProtocolModifier.length() > 0) {theSearch.query.modifier.remove("/" + oldProtocolModifier); theSearch.query.modifier.remove(oldProtocolModifier);}theSearch.query.modifier.protocol = "";theSearch.query.getQueryGoal().query_original = oldQuery.replaceAll(" /https", "").replaceAll(" /http", "").replaceAll(" /ftp", "").replaceAll(" /smb", "").replaceAll(" /file", "");@@ -235,7 +237,7 @@ public class yacysearchtrailer {prop.put(fileType, "nav-protocols_element_" + i + "_modifier", nav);url = QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false, authenticated).toString();} else {- neg++;+ neg++;prop.put("nav-protocols_element_" + i + "_on", 1);prop.put("nav-protocols_element_" + i + "_onclick", 1);prop.put(fileType, "nav-protocols_element_" + i + "_modifier", "-" + nav);@@ -280,15 +282,15 @@ public class yacysearchtrailer {if(count == 0) {continue;}- String shortname = name.substring(0, 10);- long d = Instant.parse(name).toEpochMilli();- Date dd = new Date(d);+ final String shortname = name.substring(0, 10);+ final long d = Instant.parse(name).toEpochMilli();+ final Date dd = new Date(d);if (fromconstraint != null && dd.before(fromconstraint)) continue;if (toconstraint != null && dd.after(toconstraint)) break;if (dx > 0) {while (d - dx > AbstractFormatter.dayMillis && i < theSearch.getQuery().getDateFacetMaxCount()) {dx += AbstractFormatter.dayMillis;- String sn = new Date(dx).toInstant().toString().substring(0, 10);+ final String sn = new Date(dx).toInstant().toString().substring(0, 10);prop.put("nav-dates_element_" + i + "_on", 0);prop.put(fileType, "nav-dates_element_" + i + "_name", sn);prop.put("nav-dates_element_" + i + "_count", 0);@@ -320,8 +322,8 @@ public class yacysearchtrailer {final Map<String, ScoreMap<String>> vocabularyNavigators = theSearch.vocabularyNavigator;if (vocabularyNavigators != null && !vocabularyNavigators.isEmpty()) {int navvoccount = 0;- vocnav: for (Map.Entry<String, ScoreMap<String>> ve: vocabularyNavigators.entrySet()) {- String navname = ve.getKey();+ vocnav: for (final Map.Entry<String, ScoreMap<String>> ve: vocabularyNavigators.entrySet()) {+ final String navname = ve.getKey();if (ve.getValue() == null || ve.getValue().isEmpty()) {continue vocnav;}@@ -366,9 +368,9 @@ public class yacysearchtrailer {// navigator pluginsint ni = 0;- for (String naviname : theSearch.navigatorPlugins.keySet()) {+ for (final String naviname : theSearch.navigatorPlugins.keySet()) {- Navigator navi = theSearch.navigatorPlugins.get(naviname);+ final Navigator navi = theSearch.navigatorPlugins.get(naviname);if (navi.isEmpty()) {continue;}@@ -383,16 +385,16 @@ public class yacysearchtrailer {} else {if(navi.getSort().getSortType() == NavigatorSortType.COUNT) {if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) {- navSort = 0;+ navSort = 0;} else {navSort = 1;}} else {if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) {- navSort = 2;+ navSort = 2;} else {navSort = 3;- }+ }}}prop.put("navs_" + ni + "_navSort", navSort);@@ -411,10 +413,10 @@ public class yacysearchtrailer {rawNav = navi.getQueryModifier(name);try {nav = URLEncoder.encode(rawNav, StandardCharsets.UTF_8.name());- } catch (UnsupportedEncodingException ex) {+ } catch (final UnsupportedEncodingException ex) {nav = "";}- boolean isactive = navi.modifieractive(theSearch.query.modifier, name);+ final boolean isactive = navi.modifieractive(theSearch.query.modifier, name);final String navUrl;if (!isactive) {pos++;@@ -491,8 +493,8 @@ public class yacysearchtrailer {* @param startRecord the zero based index of the first record to return* @param meanMax the maximum number of suggestions ("Did you mean") to propose*/- private static void appendSearchFormValues(final String templatePrefix, final serverObjects post, final serverObjects prop,- boolean global, final SearchEvent theSearch, final String former, final String snippetFetchStrategyName,+ private static void appendSearchFormValues(final String templatePrefix, final serverObjects post, final serverObjects prop,+ final boolean global, final SearchEvent theSearch, final String former, final String snippetFetchStrategyName,final int startRecord, final int meanMax) {prop.putHTML(templatePrefix + "former", former);prop.put(templatePrefix + "authSearch", post.containsKey("auth"));