Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

moved all remaining servlet classes to new location

Michael Peter Christenadbda4c71b38
diff --git a/htroot/AccessGrid_p.java b/source/net/yacy/htroot/AccessGrid_p.java
similarity index 98%
rename from htroot/AccessGrid_p.java
rename to source/net/yacy/htroot/AccessGrid_p.java
index 09cffd1c2..3174084b9 100644
--- a/htroot/AccessGrid_p.java
+++ b/source/net/yacy/htroot/AccessGrid_p.java
@@ -22,6 +22,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
diff --git a/htroot/AccessPicture_p.java b/source/net/yacy/htroot/AccessPicture_p.java
similarity index 97%
rename from htroot/AccessPicture_p.java
rename to source/net/yacy/htroot/AccessPicture_p.java
index e1432fc5f..782815be4 100644
--- a/htroot/AccessPicture_p.java
+++ b/source/net/yacy/htroot/AccessPicture_p.java
@@ -22,6 +22,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.util.ConcurrentModificationException;
import java.util.Date;
import java.util.Iterator;
@@ -89,10 +91,10 @@ public class AccessPicture_p {
// draw home peer
final int centerx = (picture.gridWidth() >> 1) - 1;
final int centery = picture.gridHeight() >> 1;
- long color_dot_l = Long.parseLong(color_dot, 16);
- long color_text_l = Long.parseLong(color_text, 16);
- long color_line_l = Long.parseLong(color_line, 16);
- long color_grid_l = Long.parseLong(color_grid, 16);
+ final long color_dot_l = Long.parseLong(color_dot, 16);
+ final long color_text_l = Long.parseLong(color_text, 16);
+ final long color_line_l = Long.parseLong(color_line, 16);
+ final long color_grid_l = Long.parseLong(color_grid, 16);
picture.setColor(color_dot_l);
picture.gridDot(centerx, centery, 5, true, 100);
if (corona) {
diff --git a/htroot/AccessTracker_p.java b/source/net/yacy/htroot/AccessTracker_p.java
similarity index 94%
rename from htroot/AccessTracker_p.java
rename to source/net/yacy/htroot/AccessTracker_p.java
index 07f9e422d..efceebc1a 100644
--- a/htroot/AccessTracker_p.java
+++ b/source/net/yacy/htroot/AccessTracker_p.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Collection;
import java.util.ConcurrentModificationException;
import java.util.Date;
@@ -46,9 +48,9 @@ import net.yacy.search.Switchboard;
import net.yacy.search.query.AccessTracker;
import net.yacy.search.query.QueryParams;
import net.yacy.server.serverAccessTracker;
+import net.yacy.server.serverAccessTracker.Track;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
-import net.yacy.server.serverAccessTracker.Track;
public class AccessTracker_p {
@@ -59,7 +61,7 @@ public class AccessTracker_p {
} catch (final ConcurrentModificationException e) {}
return accessClone;
}
-
+
/**
* Remove any eventual slash character at the beginning of path.
* @param path a path relative to YaCy context
@@ -125,7 +127,7 @@ public class AccessTracker_p {
prop.putHTML("page_list_" + entCount + "_host", host);
prop.put("page_list_" + entCount + "_date", GenericFormatter.SIMPLE_FORMATTER.format(new Date(entry.getTime())));
prop.putHTML("page_list_" + entCount + "_path", entry.getPath());
- /* For better integration of YaCy peers behind a reverse proxy subfolder,
+ /* For better integration of YaCy peers behind a reverse proxy subfolder,
* ensure a path relative to this servlet (with no starting slash) is used for links URLs.
* We keep the pathes starting with a slash for display only. */
prop.putHTML("page_list_" + entCount + "_relativePath", getRelativePathToThis(entry.getPath()));
@@ -148,7 +150,7 @@ public class AccessTracker_p {
prop.putHTML("page_list_" + entCount + "_host", host);
prop.put("page_list_" + entCount + "_date", GenericFormatter.SIMPLE_FORMATTER.format(new Date(entry.getTime())));
prop.putHTML("page_list_" + entCount + "_path", entry.getPath());
- /* For better integration of YaCy peers behind a reverse proxy subfolder,
+ /* For better integration of YaCy peers behind a reverse proxy subfolder,
* ensure a path relative to this servlet (with no starting slash) is used for links URLs.
* We keep the pathes starting with a slash for display only. */
prop.putHTML("page_list_" + entCount + "_relativePath", getRelativePathToThis(entry.getPath()));
@@ -172,8 +174,8 @@ public class AccessTracker_p {
long utimeSum = 0;
long stimeSum = 0;
long rtimeSum = 0;
- long utimeSum1 = 0;
- long stimeSum1 = 0;
+ final long utimeSum1 = 0;
+ final long stimeSum1 = 0;
long rtimeSum1 = 0;
int m = 0;
@@ -213,13 +215,13 @@ public class AccessTracker_p {
utimeSum += query.urlretrievaltime;
stimeSum += query.snippetcomputationtime;
rtimeSum += query.searchtime;
-
+
if (query.transmitcount > 0){
rcount++;
/* utimeSum1 += query.urlretrievaltime;
stimeSum1 += query.snippetcomputationtime; */
rtimeSum1 += query.searchtime;
- }
+ }
m++;
}
prop.put("page_list", m);
@@ -267,15 +269,15 @@ public class AccessTracker_p {
// calculate top search query words
if (page == 2 && AccessTracker.getDumpFile() != null && AccessTracker.getDumpFile().exists()) {
- Date toDate = new Date();
- Date fromDate = new Date(toDate.getTime() - 7 * 24 * 3600 * 1000); // 7 Days earlier
- List<EventTracker.Event> evList = AccessTracker.readLog(AccessTracker.getDumpFile(), fromDate, toDate);
- OrderedScoreMap<String> topicNavigator = new OrderedScoreMap<String>(String.CASE_INSENSITIVE_ORDER);
- for (EventTracker.Event ev : evList) {
- String qs = ev.payload.toString();
+ final Date toDate = new Date();
+ final Date fromDate = new Date(toDate.getTime() - 7 * 24 * 3600 * 1000); // 7 Days earlier
+ final List<EventTracker.Event> evList = AccessTracker.readLog(AccessTracker.getDumpFile(), fromDate, toDate);
+ final OrderedScoreMap<String> topicNavigator = new OrderedScoreMap<String>(String.CASE_INSENSITIVE_ORDER);
+ for (final EventTracker.Event ev : evList) {
+ final String qs = ev.payload.toString();
if (qs.startsWith("qs ")) { // currently only raw querystring "qs" lines are included
- String[] words = CommonPattern.SPACE.split(qs.substring(3));
- for (String w : words) {
+ final String[] words = CommonPattern.SPACE.split(qs.substring(3));
+ for (final String w : words) {
if (!w.isEmpty()) {
topicNavigator.inc(w);
}
@@ -294,8 +296,8 @@ public class AccessTracker_p {
prop.put("page_nav-topics", "1");
String name;
int i = 0;
- int maxcount = topicNavigator.getMaxScore();
- Iterator<String> navigatorIterator = topicNavigator.iterator();
+ final int maxcount = topicNavigator.getMaxScore();
+ final Iterator<String> navigatorIterator = topicNavigator.iterator();
while (i < TOPWORDS_MAXCOUNT && navigatorIterator.hasNext()) {
name = navigatorIterator.next();
diff --git a/htroot/Autocrawl_p.java b/source/net/yacy/htroot/Autocrawl_p.java
similarity index 98%
rename from htroot/Autocrawl_p.java
rename to source/net/yacy/htroot/Autocrawl_p.java
index 27c629a2d..02122f62b 100644
--- a/htroot/Autocrawl_p.java
+++ b/source/net/yacy/htroot/Autocrawl_p.java
@@ -1,3 +1,5 @@
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@@ -9,7 +11,7 @@ public class Autocrawl_p {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
-
+
boolean autocrawlEnable = sb.getConfigBool(SwitchboardConstants.AUTOCRAWL, false);
int autocrawlRatio = Integer.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_RATIO, "50"));
int autocrawlRows = Integer.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_ROWS, "100"));
@@ -19,7 +21,7 @@ public class Autocrawl_p {
int autocrawlDeep = Integer.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_DEEP_DEPTH, "3"));
boolean autocrawlText = sb.getConfigBool(SwitchboardConstants.AUTOCRAWL_INDEX_TEXT, true);
boolean autocrawlMedia = sb.getConfigBool(SwitchboardConstants.AUTOCRAWL_INDEX_MEDIA, true);
-
+
if (post != null) {
autocrawlEnable = post.getBoolean("autocrawlEnable");
if (post.containsKey("autocrawlRatio")) {
@@ -43,7 +45,7 @@ public class Autocrawl_p {
autocrawlText = post.getBoolean("autocrawlText");
autocrawlMedia = post.getBoolean("autocrawlMedia");
}
-
+
if (autocrawlRatio > 500) {
autocrawlRatio = 500;
} else if (autocrawlRatio < 1) {
@@ -69,7 +71,7 @@ public class Autocrawl_p {
} else if (autocrawlDeep < 1) {
autocrawlDeep = 1;
}
-
+
if (post != null) {
sb.setConfig(SwitchboardConstants.AUTOCRAWL, autocrawlEnable);
sb.setConfig(SwitchboardConstants.AUTOCRAWL_RATIO, autocrawlRatio);
@@ -80,12 +82,12 @@ public class Autocrawl_p {
sb.setConfig(SwitchboardConstants.AUTOCRAWL_DEEP_DEPTH, autocrawlDeep);
sb.setConfig(SwitchboardConstants.AUTOCRAWL_INDEX_TEXT, autocrawlText);
sb.setConfig(SwitchboardConstants.AUTOCRAWL_INDEX_MEDIA, autocrawlMedia);
-
+
sb.initAutocrawl(autocrawlEnable);
-
+
prop.put("changed", true);
}
-
+
prop.put("autocrawlEnable", autocrawlEnable);
prop.put("autocrawlRatio", autocrawlRatio);
prop.put("autocrawlRows", autocrawlRows);
@@ -95,7 +97,7 @@ public class Autocrawl_p {
prop.put("autocrawlDeep", autocrawlDeep);
prop.put("autocrawlText", autocrawlText);
prop.put("autocrawlMedia", autocrawlMedia);
-
+
return prop;
}
}
diff --git a/htroot/Banner.java b/source/net/yacy/htroot/Banner.java
similarity index 99%
rename from htroot/Banner.java
rename to source/net/yacy/htroot/Banner.java
index 4f8667edc..93c74eb70 100644
--- a/htroot/Banner.java
+++ b/source/net/yacy/htroot/Banner.java
@@ -25,6 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
diff --git a/htroot/BlacklistCleaner_p.java b/source/net/yacy/htroot/BlacklistCleaner_p.java
similarity index 98%
rename from htroot/BlacklistCleaner_p.java
rename to source/net/yacy/htroot/BlacklistCleaner_p.java
index e0a23134e..7ea0f2b22 100644
--- a/htroot/BlacklistCleaner_p.java
+++ b/source/net/yacy/htroot/BlacklistCleaner_p.java
@@ -29,6 +29,8 @@
// javac -classpath .:../classes BlacklistCleaner_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
@@ -48,9 +50,9 @@ import net.yacy.data.ListManager;
import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.repository.Blacklist;
-import net.yacy.repository.BlacklistHostAndPath;
import net.yacy.repository.Blacklist.BlacklistError;
import net.yacy.repository.Blacklist.BlacklistType;
+import net.yacy.repository.BlacklistHostAndPath;
import net.yacy.search.Switchboard;
import net.yacy.search.query.SearchEventCache;
import net.yacy.server.serverObjects;
@@ -160,7 +162,7 @@ public class BlacklistCleaner_p {
/**
* Adds a list of blacklist to the server objects properties which are used
* to display the blacklist in the HTML page belonging to this servlet.
- *
+ *
* @param prop
* Server objects properties object.
* @param lists
@@ -206,7 +208,7 @@ public class BlacklistCleaner_p {
* {@link getByPrefix(de.anomic.server.serverObjects, java.lang.String,
* boolean, boolean)} which has been created to make it easier to understand
* the code.
- *
+ *
* @param post
* All POST values.
* @param prefix
@@ -225,7 +227,7 @@ public class BlacklistCleaner_p {
* sent and returns them as an array. This method is only a wrapper for
* {@link getByPrefix(de.anomic.server.serverObjects, java.lang.String,
* boolean, boolean)}.
- *
+ *
* @param post
* All POST values.
* @param prefix
@@ -245,7 +247,7 @@ public class BlacklistCleaner_p {
* boolean)} and {@link getValuesByPrefix(de.anomic.server.serverObjects,
* java.lang.String prefix, boolean)} which have been crested to make it
* easier to understand the code.
- *
+ *
* @param post
* @param prefix
* @param useKeys
@@ -281,7 +283,7 @@ public class BlacklistCleaner_p {
/**
* Finds illegal entries in black list.
- *
+ *
* @param blacklistToUse
* The blacklist to be checked.
* @param allowRegex
@@ -292,7 +294,7 @@ public class BlacklistCleaner_p {
* error code as value.
*/
private static Map<String, BlacklistError> getIllegalEntries(
- final String blacklistToUse,
+ final String blacklistToUse,
final boolean allowRegex) {
final Map<String, BlacklistError> illegalEntries = new HashMap<>();
final Set<String> legalEntries = new HashSet<>();
@@ -325,7 +327,7 @@ public class BlacklistCleaner_p {
/**
* Removes existing entries from a blacklist.
- *
+ *
* @param blacklistToUse
* The blacklist which contains the
* @param supportedBlacklistTypes
@@ -371,7 +373,7 @@ public class BlacklistCleaner_p {
/**
* Changes existing entry in a blacklist.
- *
+ *
* @param blacklistToUse
* The blacklist which contains the entry.
* @param supportedBlacklistTypes
@@ -405,7 +407,7 @@ public class BlacklistCleaner_p {
blacklistToUse)) {
try {
Switchboard.urlBlacklist.add(s, blacklistToUse, newEntries);
- } catch (PunycodeException e) {
+ } catch (final PunycodeException e) {
ConcurrentLog.warn(APP_NAME,
"Unable to add blacklist entry to blacklist "
+ s, e);
diff --git a/htroot/BlacklistImpExp_p.java b/source/net/yacy/htroot/BlacklistImpExp_p.java
similarity index 94%
rename from htroot/BlacklistImpExp_p.java
rename to source/net/yacy/htroot/BlacklistImpExp_p.java
index 0235cd75f..3cc763b41 100644
--- a/htroot/BlacklistImpExp_p.java
+++ b/source/net/yacy/htroot/BlacklistImpExp_p.java
@@ -29,6 +29,8 @@
// javac -classpath .:../classes Blacklist_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.Iterator;
import java.util.List;
import java.util.TreeMap;
@@ -72,7 +74,7 @@ public class BlacklistImpExp_p {
if (seed != null) hostList.put(seed.get(Seed.NAME, "nameless"),seed.hash);
}
- for (String peername : hostList.keySet()) {
+ for (final String peername : hostList.keySet()) {
final String Hash = hostList.get(peername);
prop.putHTML(DISABLED + "otherHosts_" + peerCount + "_hash", Hash);
prop.putXML(DISABLED + "otherHosts_" + peerCount + "_name", peername);
@@ -89,7 +91,7 @@ public class BlacklistImpExp_p {
prop.put("disabled", (blacklistToUse == null) ? "1" : "0");
int count = 0;
- for (String element : dirlist) {
+ for (final String element : dirlist) {
prop.putHTML("blackListNames_" + count + "_blackListName", element);
count++;
}
diff --git a/htroot/BlacklistTest_p.java b/source/net/yacy/htroot/BlacklistTest_p.java
similarity index 99%
rename from htroot/BlacklistTest_p.java
rename to source/net/yacy/htroot/BlacklistTest_p.java
index 7eb96d272..5e823d2b7 100644
--- a/htroot/BlacklistTest_p.java
+++ b/source/net/yacy/htroot/BlacklistTest_p.java
@@ -29,6 +29,8 @@
// javac -classpath .:../classes Blacklist_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.net.MalformedURLException;
import net.yacy.cora.document.id.DigestURL;
diff --git a/htroot/Blacklist_p.java b/source/net/yacy/htroot/Blacklist_p.java
similarity index 98%
rename from htroot/Blacklist_p.java
rename to source/net/yacy/htroot/Blacklist_p.java
index 2218c63c1..47b9d5a40 100644
--- a/htroot/Blacklist_p.java
+++ b/source/net/yacy/htroot/Blacklist_p.java
@@ -29,6 +29,8 @@
// javac -classpath .:../classes Blacklist_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
@@ -54,13 +56,13 @@ import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
public class Blacklist_p {
-
+
private final static String EDIT = "edit_";
private final static String DISABLED = "disabled_";
private final static String BLACKLIST = "blackLists_";
private final static String BLACKLIST_MOVE = "blackListsMove_";
private final static String BLACKLIST_SHARED = "BlackLists.Shared";
-
+
public static serverObjects respond(final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
// load all blacklist files located in the directory
@@ -173,7 +175,7 @@ public class Blacklist_p {
ListManager.removeFromListSet(supportedBlacklistType + ".BlackLists",blacklistToUse);
}
}
-
+
Switchboard.urlBlacklist.clear();
ListManager.reloadBlacklists();
@@ -204,7 +206,7 @@ public class Blacklist_p {
blacklistToUse = post.get("currentBlacklist", "").trim();
final String[] selectedBlacklistEntries = post.getAll("selectedEntry.*");
-
+
// store this call as api call
ListManager.switchboard.tables.recordAPICall(post,
"Blacklist_p.html",
@@ -258,7 +260,7 @@ public class Blacklist_p {
final String targetBlacklist = post.get("targetBlacklist");
final String[] selectedBlacklistEntries = post.getAll("selectedEntry.*");
-
+
// store this call as api call
ListManager.switchboard.tables.recordAPICall(
post,
@@ -280,7 +282,7 @@ public class Blacklist_p {
prop.put(serverObjects.ACTION_LOCATION, temp);
return prop;
}
-
+
if (!BlacklistHelper.addBlacklistEntry(targetBlacklist, selectedBlacklistEntry)) {
prop.put(DISABLED + EDIT + "moveError", true);
break;
@@ -316,9 +318,9 @@ public class Blacklist_p {
for (final Entry<String, String> selectedEntry : selectedBlacklistEntries.entrySet()) {
final String editedEntryValue = editedBlacklistEntries.get(selectedEntry.getKey().replace("selectedBlacklistEntry.", "editedBlacklistEntry."));
-
+
final String preparedNewEntry = prepareNormalizedEntry(editedEntryValue);
-
+
if (!normalizeEntry(selectedEntry.getValue()).equals(preparedNewEntry)) {
/* Add first, to detect any eventual syntax errors before removing the old entry */
@@ -338,15 +340,15 @@ public class Blacklist_p {
} else {
/* At least one error occurred : display again entries with errors for edition */
prop.put(DISABLED + EDIT + "editError", true);
-
+
final int maxDisplayedErrors = 10;
-
+
int i = 0;
for (final Entry<String, String> selected2Edited : selected2EditedErrors.entrySet()) {
/* We do not use here putHTML as we don't want '+' characters to be interpreted as application/x-www-form-urlencoded encoding */
prop.put(DISABLED + EDIT + "editList_" + i + "_item", CharacterCoding.unicode2html(selected2Edited.getKey(), true));
prop.put(DISABLED + EDIT + "editList_" + i + "_count", i);
-
+
/* We do not use here putHTML as we don't want '+' characters to be interpreted as application/x-www-form-urlencoded encoding */
if(i < maxDisplayedErrors) {
prop.put(DISABLED + EDIT + "editError_list_" + i + "_item", CharacterCoding.unicode2html(selected2Edited.getValue(), true));
@@ -365,7 +367,7 @@ public class Blacklist_p {
prop.put(DISABLED + EDIT + "editList", selected2EditedErrors.size());
prop.put(DISABLED + EDIT + "editError_list", Math.min(maxDisplayedErrors, selected2EditedErrors.size()));
}
-
+
// else return entry to be edited
} else {
@@ -396,7 +398,7 @@ public class Blacklist_p {
if (blacklistToUse != null) {
int entryCount = 0;
// we should use same load methode as during internal init (even if we only need a simple array to display content)
- String[] sortedlist = BlacklistHelper.blacklistToSortedArray(blacklistToUse);
+ final String[] sortedlist = BlacklistHelper.blacklistToSortedArray(blacklistToUse);
// display them
boolean dark = true;
int offset = 0;
@@ -423,13 +425,13 @@ public class Blacklist_p {
if (nextEntry.charAt(0) == '#') {
continue;
}
-
+
/** Decode the entry for easier reading of paths with non ascii characters */
final int slashPos = nextEntry.indexOf('/', 0);
if(slashPos > 0) {
nextEntry = nextEntry.substring(0, slashPos + 1) + MultiProtocolURL.unescapePath(nextEntry.substring(slashPos + 1));
}
-
+
prop.put(DISABLED + EDIT + "Itemlist_" + entryCount + "_dark", dark ? "1" : "0");
dark = !dark;
/* We do not use here putHTML as we don't want '+' characters to be interpreted as application/x-www-form-urlencoded encoding */
@@ -440,7 +442,7 @@ public class Blacklist_p {
prop.put(DISABLED + EDIT + "Itemlist", entryCount);
// create selection of sublist
- int[] navbar = new int[7];
+ final int[] navbar = new int[7];
// generate array of start index for navbar ( -1 = early endmark)
// [0] [-2*size] [-1*size] [offset] [+1*size] [+2*size] [end]
if (size > 0 && sortedlist.length > 0) {
@@ -476,7 +478,7 @@ public class Blacklist_p {
prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_tvalue", end);
if (navbar[entryCount] == offset) {
prop.put(DISABLED + EDIT + "subListOffset_" + entryCount + "_selected", 1);
- }
+ }
entryCount++;
}
} else {
@@ -578,7 +580,7 @@ public class Blacklist_p {
* BlacklistHelper.addBlacklistEntry()
*/
}
- String path = MultiProtocolURL.escapePathPattern(entry.substring(slashPos + 1));
+ final String path = MultiProtocolURL.escapePathPattern(entry.substring(slashPos + 1));
return host + "/" + path;
}
diff --git a/htroot/Blog.java b/source/net/yacy/htroot/Blog.java
similarity index 99%
rename from htroot/Blog.java
rename to source/net/yacy/htroot/Blog.java
index 5f88bbb47..01d3f49f4 100644
--- a/htroot/Blog.java
+++ b/source/net/yacy/htroot/Blog.java
@@ -30,6 +30,8 @@
// javac -classpath .:../classes Blog.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.text.DateFormat;
import java.util.Date;
import java.util.HashMap;
diff --git a/htroot/BlogComments.java b/source/net/yacy/htroot/BlogComments.java
similarity index 98%
rename from htroot/BlogComments.java
rename to source/net/yacy/htroot/BlogComments.java
index d9137c2e9..8dbb43536 100644
--- a/htroot/BlogComments.java
+++ b/source/net/yacy/htroot/BlogComments.java
@@ -29,12 +29,16 @@
// javac -classpath .:../Classes Blacklist_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Date;
import java.util.Iterator;
+import com.google.common.io.Files;
+
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
@@ -42,10 +46,10 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.CommonPattern;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.BlogBoard;
+import net.yacy.data.BlogBoard.BlogEntry;
import net.yacy.data.BlogBoardComments;
import net.yacy.data.MessageBoard;
import net.yacy.data.UserDB;
-import net.yacy.data.BlogBoard.BlogEntry;
import net.yacy.peers.Network;
import net.yacy.peers.Seed;
import net.yacy.search.Switchboard;
@@ -53,8 +57,6 @@ import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
-import com.google.common.io.Files;
-
public class BlogComments {
@@ -86,7 +88,7 @@ public class BlogComments {
}
}
- String pagename = post.get("page", DEFAULT_PAGE);
+ final String pagename = post.get("page", DEFAULT_PAGE);
final String ip = post.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);
String strAuthor = post.get("author", "anonymous");
@@ -134,7 +136,7 @@ public class BlogComments {
sb.blogCommentDB.write(sb.blogCommentDB.newEntry(commentID, subject, author, ip, date, content));
prop.putHTML(serverObjects.ACTION_LOCATION,"BlogComments.html?page=" + pagename);
- MessageBoard.entry msgEntry = sb.messageDB.newEntry(
+ final MessageBoard.entry msgEntry = sb.messageDB.newEntry(
"blogComment",
strAuthor,
sb.peers.mySeed().hash,
diff --git a/htroot/Bookmarks.java b/source/net/yacy/htroot/Bookmarks.java
similarity index 97%
rename from htroot/Bookmarks.java
rename to source/net/yacy/htroot/Bookmarks.java
index 4c9b78d80..9ac240260 100644
--- a/htroot/Bookmarks.java
+++ b/source/net/yacy/htroot/Bookmarks.java
@@ -29,6 +29,8 @@
// javac -classpath .:../Classes Blacklist_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
@@ -48,10 +50,10 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.BookmarkHelper;
import net.yacy.data.BookmarksDB;
-import net.yacy.data.ListManager;
-import net.yacy.data.UserDB;
import net.yacy.data.BookmarksDB.Bookmark;
import net.yacy.data.BookmarksDB.Tag;
+import net.yacy.data.ListManager;
+import net.yacy.data.UserDB;
import net.yacy.document.Document;
import net.yacy.document.Parser;
import net.yacy.http.servlets.YaCyDefaultServlet;
@@ -180,7 +182,7 @@ public class Bookmarks {
}
bookmark.setTags(tags, true);
-
+
if (post.containsKey("query")) {
bookmark.setProperty(Bookmark.BOOKMARK_QUERY, post.get("query"));
bookmark.setTimeStamp(System.currentTimeMillis());
@@ -193,7 +195,7 @@ public class Bookmarks {
prop.put("mode", "2");
prop.put("display", "1");
display = 1;
- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
if (urlHash.isEmpty()) {
prop.put("mode_edit", "0"); // create mode
prop.putHTML("mode_title", post.get("title"));
@@ -204,7 +206,7 @@ public class Bookmarks {
prop.put("mode_public", "0");
prop.put("mode_feed", "0");
} else {
- BookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(urlHash);
+ final BookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(urlHash);
if (bookmark == null) {
// try to get the bookmark from the LURL database
@@ -325,7 +327,7 @@ public class Bookmarks {
count = 0;
while(count < max_count && it.hasNext()) {
- Bookmark bookmark = sb.bookmarksDB.getBookmark(it.next());
+ final Bookmark bookmark = sb.bookmarksDB.getBookmark(it.next());
if (bookmark != null){
if (bookmark.getFeed() && isAdmin) {
@@ -390,7 +392,7 @@ public class Bookmarks {
count = recurseFolders(BookmarkHelper.getFolderList("/", sb.bookmarksDB.getTagIterator(isAdmin)), "/", 0, true, "");
prop.put("display_folderlist", count);
- BusyThread bt = sb.getThread("autosearch");
+ final BusyThread bt = sb.getThread("autosearch");
if (bt != null) {
prop.put("display_autosearchrunning","1");
prop.put("display_autosearchrunning_msg", "" );
@@ -399,7 +401,7 @@ public class Bookmarks {
prop.put("display_autosearchrunning_msg", "autosearch will terminate");
prop.put("display_autosearchrunning","0");
}
- int jobs = bt.getJobCount();
+ final int jobs = bt.getJobCount();
prop.put("display_autosearchrunning_jobcount", jobs);
int cnt=0;
String qstr = "";
@@ -487,7 +489,7 @@ public class Bookmarks {
final Iterator<String> bit = sb.bookmarksDB.getBookmarksIterator(fn, isAdmin);
while (bit.hasNext()) {
- Bookmark bookmark = sb.bookmarksDB.getBookmark(bit.next());
+ final Bookmark bookmark = sb.bookmarksDB.getBookmark(bit.next());
if (bookmark == null) break;
prop.put("display_folderlist_" + count + "_folder", "<li><a href=\"" + bookmark.getUrl() + "\" title=\"" + bookmark.getDescription() + "\">" + bookmark.getTitle() + "</a></li>");
count++;
diff --git a/htroot/CacheResource_p.java b/source/net/yacy/htroot/CacheResource_p.java
similarity index 87%
rename from htroot/CacheResource_p.java
rename to source/net/yacy/htroot/CacheResource_p.java
index d4b189677..eae69d8ea 100644
--- a/htroot/CacheResource_p.java
+++ b/source/net/yacy/htroot/CacheResource_p.java
@@ -22,6 +22,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.ByteBuffer;
@@ -47,13 +49,13 @@ import net.yacy.server.servletProperties;
public class CacheResource_p {
public static Object respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
- Switchboard sb = (Switchboard) env;
+ final Switchboard sb = (Switchboard) env;
final servletProperties prop = new servletProperties();
prop.put("resource", new byte[0]);
if (post == null) return prop;
- boolean load = post.getBoolean("load");
+ final boolean load = post.getBoolean("load");
final String u = post.get("url", "");
DigestURL url;
try {
@@ -71,7 +73,7 @@ public class CacheResource_p {
final Response response = sb.loader.load(sb.loader.request(url, false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent);
responseHeader = response.getResponseHeader();
resource = response.getContent();
- } catch (IOException e) {
+ } catch (final IOException e) {
return prop;
}
} else return prop;
@@ -98,15 +100,15 @@ public class CacheResource_p {
} else {
decoderCharset = StandardCharsets.UTF_8;
}
- CharBuffer cb = decoderCharset.decode(ByteBuffer.wrap(resource));
- ByteBuffer x = StandardCharsets.UTF_16.encode(cb); // encode to a default java string (which uses utf-16 and is handled correct for servlet content)
+ final CharBuffer cb = decoderCharset.decode(ByteBuffer.wrap(resource));
+ final ByteBuffer x = StandardCharsets.UTF_16.encode(cb); // encode to a default java string (which uses utf-16 and is handled correct for servlet content)
prop.put("resource", x.asCharBuffer().toString());
} else {
prop.put("resource", resource);
}
- // add resource
+ // add resource
return prop;
}
}
diff --git a/htroot/Collage.java b/source/net/yacy/htroot/Collage.java
similarity index 99%
rename from htroot/Collage.java
rename to source/net/yacy/htroot/Collage.java
index 95233098c..f5aa8c0ed 100644
--- a/htroot/Collage.java
+++ b/source/net/yacy/htroot/Collage.java
@@ -22,6 +22,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Random;
import net.yacy.cora.document.id.MultiProtocolURL;
diff --git a/htroot/ConfigAccountList_p.java b/source/net/yacy/htroot/ConfigAccountList_p.java
similarity index 91%
rename from htroot/ConfigAccountList_p.java
rename to source/net/yacy/htroot/ConfigAccountList_p.java
index ea772af4b..7a3dff960 100644
--- a/htroot/ConfigAccountList_p.java
+++ b/source/net/yacy/htroot/ConfigAccountList_p.java
@@ -18,6 +18,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Iterator;
import net.yacy.cora.date.GenericFormatter;
@@ -75,8 +77,8 @@ public class ConfigAccountList_p {
long percent;
if (entry.getTrafficLimit() != null) {
- long limit = entry.getTrafficLimit();
- long used = entry.getTrafficSize();
+ final long limit = entry.getTrafficLimit();
+ final long used = entry.getTrafficSize();
percent = used * 100 / limit;
prop.put("userlist_" + numUsers + "_time", percent);
} else {
@@ -85,8 +87,8 @@ public class ConfigAccountList_p {
percent = -1;
if (entry.getTimeLimit() > 0) {
- long limit = entry.getTimeLimit();
- long used = entry.getTimeUsed();
+ final long limit = entry.getTimeLimit();
+ final long used = entry.getTimeUsed();
percent = used * 100 / limit;
prop.put("userlist_" + numUsers + "_traffic", percent);
} else {
diff --git a/htroot/ConfigAccounts_p.java b/source/net/yacy/htroot/ConfigAccounts_p.java
similarity index 95%
rename from htroot/ConfigAccounts_p.java
rename to source/net/yacy/htroot/ConfigAccounts_p.java
index cd5562999..053ed4c68 100644
--- a/htroot/ConfigAccounts_p.java
+++ b/source/net/yacy/htroot/ConfigAccounts_p.java
@@ -29,6 +29,8 @@
//javac -classpath .:../Classes Message.java
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.EnumMap;
import java.util.HashMap;
import java.util.Iterator;
@@ -46,14 +48,14 @@ import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
public class ConfigAccounts_p {
-
+
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
-
+
/* Acquire a transaction token for the next POST form submission */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));
-
+
final Switchboard sb = (Switchboard) env;
UserDB.Entry entry = null;
@@ -64,7 +66,7 @@ public class ConfigAccounts_p {
TransactionManager.checkPostTransaction(header, post);
sb.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_All_PAGES, post.getBoolean(SwitchboardConstants.ADMIN_ACCOUNT_All_PAGES));
}
-
+
if (post != null && post.containsKey("setAdmin")) {
TransactionManager.checkPostTransaction(header, post);
localhostAccess = post.get("access", "").equals("localhost");
@@ -74,13 +76,13 @@ public class ConfigAccounts_p {
int inputerror=0;
// may be overwritten if new password is given
if (user.length() > 0 && pw1.equals(pw2)) {
- String oldusername = env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,user);
+ final String oldusername = env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,user);
// check passed. set account:
// old: // env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, Digest.encodeMD5Hex(Base64Order.standardCoder.encodeString(user + ":" + pw1)));
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, sb.encodeDigestAuth(user, pw1));
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,user);
// make sure server accepts new credentials
- YaCyHttpServer jhttpserver = (YaCyHttpServer)sb.getHttpServer();
+ final YaCyHttpServer jhttpserver = (YaCyHttpServer)sb.getHttpServer();
if (!user.equals(oldusername)) jhttpserver.removeUser(oldusername);
jhttpserver.resetUser(user);
} else {
@@ -91,7 +93,7 @@ public class ConfigAccounts_p {
inputerror = 2; // password match error
}
prop.put("error", inputerror);
- }
+ }
}
if (inputerror == 0) {
@@ -104,7 +106,7 @@ public class ConfigAccounts_p {
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "");
}
}
- }
+ }
}
if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").isEmpty() && !env.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false)) {
@@ -259,7 +261,7 @@ public class ConfigAccounts_p {
prop.putHTML("text_username", username);
prop.put("text", "2");
}//edit user
-
+
prop.putHTML("username", username);
if (entry != null) {
//TODO: set username read-only in html
diff --git a/htroot/ConfigAppearance_p.java b/source/net/yacy/htroot/ConfigAppearance_p.java
similarity index 93%
rename from htroot/ConfigAppearance_p.java
rename to source/net/yacy/htroot/ConfigAppearance_p.java
index ea5bae159..be32583fb 100644
--- a/htroot/ConfigAppearance_p.java
+++ b/source/net/yacy/htroot/ConfigAppearance_p.java
@@ -29,6 +29,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
@@ -39,6 +41,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import com.google.common.io.Files;
+
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.RequestHeader;
@@ -48,8 +52,6 @@ import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
-import com.google.common.io.Files;
-
public class ConfigAppearance_p {
@@ -70,7 +72,7 @@ public class ConfigAppearance_p {
}
if (post != null) {
- String selectedSkin = post.get("skin");
+ final String selectedSkin = post.get("skin");
if (post.containsKey("use_button") && selectedSkin != null) {
/* Only change skin if filename is contained in list of filesnames
@@ -138,7 +140,7 @@ public class ConfigAppearance_p {
}
}
- String currentSkin = env.getConfig("currentSkin", "default");
+ final String currentSkin = env.getConfig("currentSkin", "default");
prop.putHTML("currentskin", currentSkin);
// reread skins
skinFiles = FileUtils.getDirListing(skinPath, SKIN_FILENAME_FILTER);
@@ -146,7 +148,7 @@ public class ConfigAppearance_p {
int count = 0;
for (final String skinFile : skinFiles) {
if (skinFile.endsWith(".css")) {
- String skinName = skinFile.substring(0, skinFile.length() - 4);
+ final String skinName = skinFile.substring(0, skinFile.length() - 4);
prop.put("skinlist_" + count + "_file", skinFile);
prop.put("skinlist_" + count + "_name", skinName);
prop.put("skinlist_" + count + "_selected", skinName.equals(currentSkin) ? 1 : 0);
@@ -156,7 +158,7 @@ public class ConfigAppearance_p {
prop.put("skinlist", count);
// write colors from generic skin
- Iterator<String> i = env.configKeys();
+ final Iterator<String> i = env.configKeys();
while (i.hasNext()) {
final String key = i.next();
if (key.startsWith("color_")) prop.put(key, env.getConfig(key, "#000000").substring(1));
diff --git a/htroot/ConfigBasic.java b/source/net/yacy/htroot/ConfigBasic.java
similarity index 97%
rename from htroot/ConfigBasic.java
rename to source/net/yacy/htroot/ConfigBasic.java
index 16520dc0b..40b0a2b58 100644
--- a/htroot/ConfigBasic.java
+++ b/source/net/yacy/htroot/ConfigBasic.java
@@ -28,6 +28,8 @@
// javac -classpath .:../classes ConfigBasic_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -58,7 +60,7 @@ public class ConfigBasic {
private static final int NEXTSTEP_PEERNAME = 2;
private static final int NEXTSTEP_PEERPORT = 3;
private static final int NEXTSTEP_RECONNECT = 4;
-
+
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws FileNotFoundException, IOException {
// return variable that accumulates replacements
@@ -66,17 +68,17 @@ public class ConfigBasic {
final serverObjects prop = new serverObjects();
final File langPath = new File(sb.getAppPath("locale.source", "locales").getAbsolutePath());
String lang = env.getConfig("locale.language", "browser");
-
+
final int authentication = sb.adminAuthenticated(header);
if (authentication < 2) {
// must authenticate
prop.authenticationRequired();
return prop;
}
-
+
/* For authenticated users only : acquire a transaction token for the next POST form submission */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));
-
+
if ((sb.peers.mySeed().isVirgin()) || (sb.peers.mySeed().isJunior())) {
new OnePeerPingBusyThread(sb.yc).start();
}
@@ -93,23 +95,23 @@ public class ConfigBasic {
if(post.containsKey("set")) {
sb.tables.recordAPICall(post, "ConfigBasic.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "basic settings");
}
-
+
// language settings
if(post.containsKey("language") && !lang.equals(post.get("language", "default"))) {
if(new TranslatorXliff().changeLang(env, langPath, post.get("language", "default") + ".lng")) {
prop.put("changedLanguage", "1");
}
}
-
+
// port settings
if(post.getInt("port", 0) > 1023) {
port = post.getLong("port", 8090);
ssl = post.getBoolean("withssl");
}
-
+
// peer name settings
peerName = post.get("peername", "");
-
+
// UPnP config
if(post.containsKey("port")) { // hack to allow checkbox
upnp = post.containsKey("enableUpnp");
@@ -122,11 +124,11 @@ public class ConfigBasic {
}
}
}
-
+
if (ssl) {
prop.put("withsslenabled_sslport", env.getHttpServer().getSslPort());
}
-
+
if (peerName != null && peerName.length() > 0) {
peerName = peerName.replace(' ', '-');
}
@@ -147,7 +149,7 @@ public class ConfigBasic {
final YaCyHttpServer theServerCore = env.getHttpServer();
env.setConfig(SwitchboardConstants.SERVER_PORT, port);
env.setConfig("server.https", ssl);
-
+
// redirect the browser to the new port
reconnect = true;
@@ -166,7 +168,7 @@ public class ConfigBasic {
prop.put("nextStep_host", host);
prop.put("reconnect_port", port);
prop.put("nextStep_port", port);
- prop.put("reconnect_sslSupport", theServerCore.withSSL() ? "1" : "0");
+ prop.put("reconnect_sslSupport", theServerCore.withSSL() ? "1" : "0");
prop.put("nextStep_sslSupport", theServerCore.withSSL() ? "1" : "0");
// generate new shortcut (used for Windows)
@@ -187,8 +189,8 @@ public class ConfigBasic {
if (post != null && post.containsKey("usecase")) {
/* Settings will be modified : check this is a valid transaction using HTTP POST method */
TransactionManager.checkPostTransaction(header, post);
-
- boolean hasNonEmptyRemoteSolr = sb.index.fulltext().connectedRemoteSolr()
+
+ final boolean hasNonEmptyRemoteSolr = sb.index.fulltext().connectedRemoteSolr()
&& (sb.index.fulltext().collectionSize() > 0 || sb.index.fulltext().webgraphSize() > 0);
if ("freeworld".equals(post.get("usecase", "")) && !"freeworld".equals(networkName)) {
if("intranet".equals(networkName) && hasNonEmptyRemoteSolr) {
@@ -198,7 +200,7 @@ public class ConfigBasic {
if(hasNonEmptyRemoteSolr) {
/* One or more non empty remote Solr(s) attached : warn the user even if not coming from intranet as indexed documents may be irrelevant for the new mode */
prop.put("setUseCase_switchWarning", 2);
- }
+ }
// switch to freeworld network
sb.setConfig(SwitchboardConstants.CORE_SERVICE_RWI, true);
sb.switchNetwork("defaults/yacy.network.freeworld.unit");
@@ -219,8 +221,8 @@ public class ConfigBasic {
if(hasNonEmptyRemoteSolr) {
/* One or more non empty remote Solr(s) attached : warn the user even if not coming from intranet as indexed documents may be irrelevant for the new mode */
prop.put("setUseCase_switchWarning", 2);
- }
-
+ }
+
// switch to webportal network
sb.setConfig(SwitchboardConstants.CORE_SERVICE_RWI, false);
sb.switchNetwork("defaults/yacy.network.webportal.unit");
@@ -237,7 +239,7 @@ public class ConfigBasic {
if(hasNonEmptyRemoteSolr) {
/* One or more non empty remote Solr(s) attached : warn the user as the indexed documents may be public external resources out of scope for intranet/localhost domain. */
prop.put("setUseCase_switchWarning", 1);
- }
+ }
// switch to intranet network
sb.setConfig(SwitchboardConstants.CORE_SERVICE_RWI, false);
sb.switchNetwork("defaults/yacy.network.intranet.unit");
@@ -324,7 +326,7 @@ public class ConfigBasic {
}
// set label class (green background) for active translation
if (lang.equals("browser")) {
- List<String> l = Translator.activeTranslations();
+ final List<String> l = Translator.activeTranslations();
prop.put("active_zh", l.contains("zh") ? "2" : "1");
prop.put("active_de", l.contains("de") ? "2" : "1");
prop.put("active_fr", l.contains("fr") ? "2" : "1");
@@ -336,7 +338,7 @@ public class ConfigBasic {
prop.put("active_uk", l.contains("uk") ? "2" : "1");
prop.put("active_es", l.contains("es") ? "2" : "1");
prop.put("active_en", "2");
-
+
} else {
prop.put("active_de", "0");
prop.put("active_fr", "0");
diff --git a/htroot/ConfigHTCache_p.java b/source/net/yacy/htroot/ConfigHTCache_p.java
similarity index 98%
rename from htroot/ConfigHTCache_p.java
rename to source/net/yacy/htroot/ConfigHTCache_p.java
index 827dab17a..9608074ff 100644
--- a/htroot/ConfigHTCache_p.java
+++ b/source/net/yacy/htroot/ConfigHTCache_p.java
@@ -26,6 +26,8 @@
// javac -classpath .:../classes ProxyIndexingMonitor_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.text.NumberFormat;
@@ -63,14 +65,14 @@ public class ConfigHTCache_p {
final int newProxyCacheSize = Math.max(post.getInt("maxCacheSize", 64), 0);
env.setConfig(SwitchboardConstants.PROXY_CACHE_SIZE, newProxyCacheSize);
Cache.setMaxCacheSize(newProxyCacheSize * 1024L * 1024L);
-
+
/* Compression level*/
/* Ensure a value within the range supported by the Deflater class */
final int newCompressionLevel = Math.max(Deflater.NO_COMPRESSION, Math.min(Deflater.BEST_COMPRESSION,
post.getInt("compressionLevel", SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT)));
env.setConfig(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL, newCompressionLevel);
Cache.setCompressionLevel(newCompressionLevel);
-
+
/* Synchronization lock timeout */
final long newLockTimeout = Math.max(10, Math.min(60000,
post.getLong("lockTimeout", SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT)));
@@ -88,21 +90,21 @@ public class ConfigHTCache_p {
}
prop.put("HTCachePath", env.getConfig(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT));
-
+
/* Compression levels */
final int configuredCompressionLevel = env.getConfigInt(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL,
SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT);
int levelsCount = 0;
for(int level = Deflater.NO_COMPRESSION; level <= Deflater.BEST_COMPRESSION; level++) {
if(level == configuredCompressionLevel) {
- prop.put("compressionLevels_" + levelsCount + "_selected", "1");
+ prop.put("compressionLevels_" + levelsCount + "_selected", "1");
} else {
prop.put("compressionLevels_" + levelsCount + "_selected", "0");
}
prop.put("compressionLevels_" + levelsCount + "_value", level);
prop.put("compressionLevels_" + levelsCount + "_name", level);
if(level == Deflater.NO_COMPRESSION) {
- prop.put("compressionLevels_" + levelsCount + "_name", "0 - No compression");
+ prop.put("compressionLevels_" + levelsCount + "_name", "0 - No compression");
} else if(level == Deflater.BEST_SPEED) {
prop.put("compressionLevels_" + levelsCount + "_name", Deflater.BEST_SPEED + " - Best speed");
} else if(level == Deflater.BEST_COMPRESSION) {
@@ -111,7 +113,7 @@ public class ConfigHTCache_p {
levelsCount++;
}
prop.put("compressionLevels", levelsCount);
-
+
prop.put("lockTimeout", env.getConfigLong(SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT,
SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT));
prop.put("actualCacheSize", Cache.getActualCacheSize() / 1024 / 1024);
@@ -124,7 +126,7 @@ public class ConfigHTCache_p {
prop.put("hits", hits);
prop.put("requests", totalRequests);
prop.put("hitRate", NumberFormat.getPercentInstance().format(Cache.getHitRate()));
-
+
// return rewrite properties
return prop;
}
diff --git a/htroot/ConfigHeuristics_p.java b/source/net/yacy/htroot/ConfigHeuristics_p.java
similarity index 95%
rename from htroot/ConfigHeuristics_p.java
rename to source/net/yacy/htroot/ConfigHeuristics_p.java
index f9c549615..90fb7536b 100644
--- a/htroot/ConfigHeuristics_p.java
+++ b/source/net/yacy/htroot/ConfigHeuristics_p.java
@@ -26,16 +26,9 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-import java.io.File;
-
-import net.yacy.cora.protocol.ClientIdentification;
-import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.cora.storage.Configuration;
-import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.crawler.robots.RobotsTxtEntry;
-import net.yacy.data.WorkTables;
-import net.yacy.search.Switchboard;
+package net.yacy.htroot;
+import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Iterator;
@@ -43,9 +36,15 @@ import java.util.Locale;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.FederateSearchManager;
-
import net.yacy.cora.federate.solr.SchemaConfiguration;
+import net.yacy.cora.protocol.ClientIdentification;
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.storage.Configuration;
import net.yacy.cora.storage.Files;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.crawler.robots.RobotsTxtEntry;
+import net.yacy.data.WorkTables;
+import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.schema.WebgraphSchema;
import net.yacy.server.serverObjects;
@@ -89,7 +88,7 @@ public class ConfigHeuristics_p {
} else {
osderrmsg = "Error: webgraph Solr index not enabled";
}
- } else {
+ } else {
osderrmsg = "Solr webgraph index needs to be available and fields target_rel_s, target_protocol_s, target_urlstub_s on";
}
}
@@ -106,7 +105,7 @@ public class ConfigHeuristics_p {
/* Check eventual robots.txt policy */
RobotsTxtEntry robotsEntry = null;
try {
- MultiProtocolURL templateURL = new MultiProtocolURL(tmpurl);
+ final MultiProtocolURL templateURL = new MultiProtocolURL(tmpurl);
if (sb.robots != null) {
robotsEntry = sb.robots.getEntry(templateURL,
@@ -134,10 +133,10 @@ public class ConfigHeuristics_p {
writeopensearchcfg (sb,post);
}
- if (post.containsKey("switchsolrfieldson")) {
- final boolean metafieldavailable = sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.target_rel_s.name())
+ if (post.containsKey("switchsolrfieldson")) {
+ final boolean metafieldavailable = sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.target_rel_s.name())
&& ( sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.target_protocol_s.name()) && sb.index.fulltext().getWebgraphConfiguration().contains(WebgraphSchema.target_urlstub_s.name()) ) ;
-
+
if (!metafieldavailable) {
SchemaConfiguration.Entry entry;
entry = sb.index.fulltext().getWebgraphConfiguration().get(WebgraphSchema.target_rel_s.name());
@@ -156,7 +155,7 @@ public class ConfigHeuristics_p {
sb.index.fulltext().getWebgraphConfiguration().commit();
} catch (final IOException e) {
}
- sb.setConfig(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, true);
+ sb.setConfig(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, true);
}
}
@@ -168,7 +167,7 @@ public class ConfigHeuristics_p {
if ((post.containsKey("resettodefaultosdlist") || !osdConfig.exists()) && osdDefaultConfig.exists()) {
try {
Files.copy(osdDefaultConfig, osdConfig);
- File defdir = new File(sb.dataPath, "DATA/SETTINGS/federatecfg");
+ final File defdir = new File(sb.dataPath, "DATA/SETTINGS/federatecfg");
if (!defdir.exists()) {
Files.copy(new File(sb.appPath, "defaults/federatecfg"), defdir);
}
@@ -192,12 +191,12 @@ public class ConfigHeuristics_p {
final File f = new File (sb.getDataPath(),"DATA/SETTINGS/heuristicopensearch.conf");
try {
- Configuration p = new Configuration(f);
+ final Configuration p = new Configuration(f);
int c = 0;
boolean dark = false;
- Iterator<Configuration.Entry> i = p.entryIterator();
+ final Iterator<Configuration.Entry> i = p.entryIterator();
while (i.hasNext()) {
- SchemaConfiguration.Entry e = i.next();
+ final SchemaConfiguration.Entry e = i.next();
prop.put("osdcfg_" + c + "_dark", dark ? 1 : 0);
dark = !dark;
prop.put("osdcfg_" + c + "_checked", e.enabled() ? 1 : 0);
@@ -230,7 +229,7 @@ public class ConfigHeuristics_p {
final File f = new File(sb.getDataPath(), "DATA/SETTINGS/heuristicopensearch.conf");
try {
- Configuration cfg = new Configuration(f);
+ final Configuration cfg = new Configuration(f);
final Iterator<Configuration.Entry> cfgentries = cfg.entryIterator();
Configuration.Entry entry;
boolean modified = false; // flag to remember changes
@@ -267,7 +266,7 @@ public class ConfigHeuristics_p {
} catch (final IOException e) {
ConcurrentLog.warn("OpenSearchConnector", "file not found " + f.getAbsolutePath());
}
-
+
// re-read config (and create/update work table)
if (sb.getConfigBool(SwitchboardConstants.HEURISTIC_OPENSEARCH, true)) {
FederateSearchManager.getManager().init(f.getAbsolutePath());
diff --git a/htroot/ConfigLanguage_p.java b/source/net/yacy/htroot/ConfigLanguage_p.java
similarity index 95%
rename from htroot/ConfigLanguage_p.java
rename to source/net/yacy/htroot/ConfigLanguage_p.java
index 8851e9121..0aea2f984 100644
--- a/htroot/ConfigLanguage_p.java
+++ b/source/net/yacy/htroot/ConfigLanguage_p.java
@@ -29,7 +29,8 @@
//javac -classpath .:../Classes Blacklist_p.java
//if the shell's current path is HTROOT
-import com.google.common.io.Files;
+package net.yacy.htroot;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -40,6 +41,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import com.google.common.io.Files;
+
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.RequestHeader;
@@ -57,7 +60,7 @@ public class ConfigLanguage_p {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
- Switchboard sb = (Switchboard) env;
+ final Switchboard sb = (Switchboard) env;
final File langPath = new File(sb.getAppPath("locale.source", "locales").getAbsolutePath());
//Fallback
@@ -105,7 +108,7 @@ public class ConfigLanguage_p {
try {
final DigestURL u = new DigestURL(url);
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, null, null));
- TranslatorXliff tx = new TranslatorXliff();
+ final TranslatorXliff tx = new TranslatorXliff();
File langFile = tx.getScratchFile(new File(langPath, u.getFileName()));
try {
try (
@@ -116,16 +119,16 @@ public class ConfigLanguage_p {
bw.write(it.next() + "\n");
}
}
-
-
+
+
// convert downloaded xliff to internal lng file
final String ext = Files.getFileExtension(langFile.getName());
if (ext.equalsIgnoreCase("xlf") || ext.equalsIgnoreCase("xliff")) {
- Map<String,Map<String,String>> lng = tx.loadTranslationsListsFromXliff(langFile);
+ final Map<String,Map<String,String>> lng = tx.loadTranslationsListsFromXliff(langFile);
langFile = new File(langPath, Files.getNameWithoutExtension(langFile.getName())+".lng");
tx.saveAsLngFile(null, langFile, lng);
}
-
+
if (post.containsKey("use_lang") && "on".equals(post.get("use_lang"))) {
tx.changeLang(env, langPath, langFile.getName());
}
@@ -153,7 +156,7 @@ public class ConfigLanguage_p {
prop.put("langlist_1_name", ((langNames.get("default") == null) ? "default" : langNames.get("default")));
prop.put("langlist_1_selected", sellang.equals("default") ? "selected=\"selected\"":" ");
int count = 2; //+2 because of the virtual entry "browser" and "default" at top
- for (final String langFile : langFiles) {
+ for (final String langFile : langFiles) {
final String langKey = langFile.substring(0, langFile.length() -4);
final String langName = langNames.get(langKey);
prop.put("langlist_" + (count) + "_file", langFile);
diff --git a/htroot/ConfigNetwork_p.java b/source/net/yacy/htroot/ConfigNetwork_p.java
similarity index 98%
rename from htroot/ConfigNetwork_p.java
rename to source/net/yacy/htroot/ConfigNetwork_p.java
index 1f368149d..2ff40ec92 100644
--- a/htroot/ConfigNetwork_p.java
+++ b/source/net/yacy/htroot/ConfigNetwork_p.java
@@ -25,6 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -52,10 +54,10 @@ public class ConfigNetwork_p
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
-
+
/* Acquire a transaction token for the next POST form submission */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));
-
+
int commit = 0;
// load all options for network definitions
@@ -75,7 +77,7 @@ public class ConfigNetwork_p
if ( post.containsKey("changeNetwork") ) {
/* Settings will be modified : check this is a valid transaction using HTTP POST method */
TransactionManager.checkPostTransaction(header, post);
-
+
String networkDefinition = post.get("networkDefinition", "defaults/yacy.network.freeworld.unit");
final String networkDefinitionURL = post.get("networkDefinitionURL", "");
if ( !networkDefinitionURL.equals("")) {
@@ -91,7 +93,7 @@ public class ConfigNetwork_p
sb.switchNetwork(networkDefinition);
}
}
-
+
if (post.containsKey("setEncryption")) {
/*
* Settings will be modified : check this is a valid transaction using HTTP POST
@@ -119,7 +121,7 @@ public class ConfigNetwork_p
// DHT control
boolean indexDistribute = "on".equals(post.get("indexDistribute", ""));
boolean indexReceive = "on".equals(post.get("indexReceive", ""));
- boolean indexReceiveSearch = "on".equals(post.get("indexReceiveSearch", ""));
+ final boolean indexReceiveSearch = "on".equals(post.get("indexReceiveSearch", ""));
if ( !indexReceive ) {
// remove heuristics
sb.setConfig(SwitchboardConstants.HEURISTIC_SITE, false);
@@ -194,7 +196,7 @@ public class ConfigNetwork_p
prop.put("indexDistributeChecked", indexDistribute);
prop.put("indexReceiveChecked", indexReceive);
prop.put("indexReceiveSearchChecked", indexReceiveSearch);
-
+
// set seed information directly
sb.peers.mySeed().setFlagAcceptRemoteCrawl(sb.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false));
sb.peers.mySeed().setFlagAcceptRemoteIndex(indexReceive);
@@ -231,7 +233,7 @@ public class ConfigNetwork_p
prop.put("networks_" + c++ + "_network", s);
}
prop.put("networks", c);
-
+
// set encryption setting
prop.put(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
diff --git a/htroot/ConfigParser_p.java b/source/net/yacy/htroot/ConfigParser_p.java
similarity index 99%
rename from htroot/ConfigParser_p.java
rename to source/net/yacy/htroot/ConfigParser_p.java
index 1294cfba4..e466d783b 100644
--- a/htroot/ConfigParser_p.java
+++ b/source/net/yacy/htroot/ConfigParser_p.java
@@ -25,6 +25,8 @@
// javac -classpath .:../Classes Settings_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.Parser;
import net.yacy.document.TextParser;
diff --git a/htroot/ConfigPortal.java b/source/net/yacy/htroot/ConfigPortal.java
similarity index 98%
rename from htroot/ConfigPortal.java
rename to source/net/yacy/htroot/ConfigPortal.java
index adeb9ab7c..a72abd910 100644
--- a/htroot/ConfigPortal.java
+++ b/source/net/yacy/htroot/ConfigPortal.java
@@ -25,6 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
diff --git a/htroot/ConfigPortal_p.java b/source/net/yacy/htroot/ConfigPortal_p.java
similarity index 97%
rename from htroot/ConfigPortal_p.java
rename to source/net/yacy/htroot/ConfigPortal_p.java
index 8b1af4c2a..1cc28987f 100644
--- a/htroot/ConfigPortal_p.java
+++ b/source/net/yacy/htroot/ConfigPortal_p.java
@@ -25,6 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.net.MalformedURLException;
import java.time.Duration;
import java.util.Properties;
@@ -52,9 +54,9 @@ public class ConfigPortal_p {
if (post != null) {
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(header, post);
-
+
boolean cleanSearchCache = false;
-
+
if (post.containsKey("popup")) {
final String popup = post.get("popup", "status");
if ("front".equals(popup)) {
@@ -88,24 +90,24 @@ public class ConfigPortal_p {
sb.setConfig("publicTopmenu", !post.containsKey("publicTopmenu") || post.getBoolean("publicTopmenu"));
sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, !post.containsKey(SwitchboardConstants.PUBLIC_SEARCHPAGE) || post.getBoolean(SwitchboardConstants.PUBLIC_SEARCHPAGE));
sb.setConfig("search.options", post.getBoolean("search.options"));
-
+
final boolean oldJsResort = sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT);
final boolean newJsResort = post.getBoolean(SwitchboardConstants.SEARCH_JS_RESORT);
/* When this setting has changed we must clean up the search event cache as it affects how search results are retrieved */
cleanSearchCache = cleanSearchCache || oldJsResort != newJsResort;
sb.setConfig(SwitchboardConstants.SEARCH_JS_RESORT, newJsResort);
-
+
final boolean oldStrictContentDom = sb.getConfigBool(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT);
final boolean newStrictContentDom = post.getBoolean(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM);
/* When this setting has changed we must clean up the search event cache as it affects how search results are retrieved */
cleanSearchCache = cleanSearchCache || oldStrictContentDom != newStrictContentDom;
sb.setConfig(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, newStrictContentDom);
-
+
sb.setConfig(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
post.getBoolean(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED));
sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, post.getBoolean(SwitchboardConstants.GREEDYLEARNING_ACTIVE));
-
+
final boolean storeresult = post.getBoolean(SwitchboardConstants.REMOTESEARCH_RESULT_STORE);
sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, storeresult);
sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, post.getLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1));
@@ -116,11 +118,11 @@ public class ConfigPortal_p {
sb.setConfig("about.headline", post.get("about.headline", ""));
sb.setConfig("about.body", post.get("about.body", ""));
- String excludehosts = post.get("search.excludehosts", "");
+ final String excludehosts = post.get("search.excludehosts", "");
sb.setConfig("search.excludehosts", excludehosts);
try {
sb.setConfig("search.excludehosth", DigestURL.hosthashes(excludehosts));
- } catch (MalformedURLException e) {
+ } catch (final MalformedURLException e) {
ConcurrentLog.logException(e);
sb.setConfig("search.excludehosth", "");
}
@@ -143,23 +145,23 @@ public class ConfigPortal_p {
sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, config.getProperty(SwitchboardConstants.PUBLIC_SEARCHPAGE,"true"));
sb.setConfig("search.navigation", config.getProperty("search.navigation","hosts,authors,namespace,topics"));
sb.setConfig("search.options", config.getProperty("search.options","true"));
-
+
final boolean oldJsResort = sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT);
final boolean newJsResort = Boolean.parseBoolean(config.getProperty(SwitchboardConstants.SEARCH_JS_RESORT, String.valueOf(SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT)));
/* When this setting has changed we must clean up the search event cache as it affects how search results are retrieved */
cleanSearchCache = cleanSearchCache || oldJsResort != newJsResort;
sb.setConfig(SwitchboardConstants.SEARCH_JS_RESORT, newJsResort);
-
+
final boolean oldStrictContentDom = sb.getConfigBool(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT);
final boolean newStrictContentDom = Boolean.parseBoolean(config.getProperty(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, String.valueOf(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT)));
/* When this setting has changed we must clean up the search event cache as it affects how search results are retrieved */
cleanSearchCache = cleanSearchCache || oldStrictContentDom != newStrictContentDom;
sb.setConfig(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM, newStrictContentDom);
-
+
sb.setConfig(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
Boolean.parseBoolean(config.getProperty(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
String.valueOf(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT))));
-
+
sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, config.getProperty(SwitchboardConstants.GREEDYLEARNING_ACTIVE));
sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE));
sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE));
@@ -170,12 +172,12 @@ public class ConfigPortal_p {
sb.setConfig("search.excludehosts", config.getProperty("search.excludehosts",""));
sb.setConfig("search.excludehosth", config.getProperty("search.excludehosth",""));
}
-
+
if(cleanSearchCache) {
SearchEventCache.cleanupEvents(true);
}
}
-
+
/* Acquire a transaction token for the next POST form submission */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));
@@ -192,11 +194,11 @@ public class ConfigPortal_p {
prop.put(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM,
sb.getConfigBool(SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM,
SwitchboardConstants.SEARCH_STRICT_CONTENT_DOM_DEFAULT) ? 1 : 0);
-
+
prop.put(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
sb.getConfigBool(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT) ? 1 : 0);
-
+
final boolean textSnippetsStatisticsEnabled = sb.getConfigBool(
SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED,
SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED_DEFAULT);
@@ -204,15 +206,15 @@ public class ConfigPortal_p {
prop.put(SwitchboardConstants.GREEDYLEARNING_ACTIVE, sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ACTIVE, false) ? 1 : 0);
prop.put(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, sb.getConfig(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, "0"));
-
+
prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, sb.getConfigBool(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, true) ? 1 : 0);
- long resultStoredMaxSize = sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1);
+ final long resultStoredMaxSize = sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1);
if(resultStoredMaxSize > 0) {
prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, resultStoredMaxSize);
} else {
prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, "");
}
-
+
/* Provide some basic stats about text snippets generation time to help choosing snippet options */
if(textSnippetsStatisticsEnabled) {
final long totalSnippets = TextSnippet.statistics.getTotalSnippets();
@@ -290,7 +292,7 @@ public class ConfigPortal_p {
*/
private static String formatDuration(final long durationValue) {
final Duration duration = Duration.ofMillis(durationValue);
-
+
final String formattedDuration;
if(duration.getSeconds() > 0) {
formattedDuration = duration.getSeconds() + "s";
diff --git a/htroot/ConfigProfile_p.java b/source/net/yacy/htroot/ConfigProfile_p.java
similarity index 97%
rename from htroot/ConfigProfile_p.java
rename to source/net/yacy/htroot/ConfigProfile_p.java
index 658046448..60f97f5d0 100644
--- a/htroot/ConfigProfile_p.java
+++ b/source/net/yacy/htroot/ConfigProfile_p.java
@@ -29,6 +29,8 @@
// javac -classpath .:../classes EditProfile_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
diff --git a/htroot/ConfigProperties_p.java b/source/net/yacy/htroot/ConfigProperties_p.java
similarity index 96%
rename from htroot/ConfigProperties_p.java
rename to source/net/yacy/htroot/ConfigProperties_p.java
index 4474b8ddb..b30a786b9 100644
--- a/htroot/ConfigProperties_p.java
+++ b/source/net/yacy/htroot/ConfigProperties_p.java
@@ -28,6 +28,8 @@
// javac -classpath .:../classes Config_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
@@ -43,7 +45,7 @@ public class ConfigProperties_p {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
-
+
/* Acquire a transaction token for the next POST form submission */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));
@@ -54,7 +56,7 @@ public class ConfigProperties_p {
if (post != null && post.containsKey("key") && post.containsKey("value")) {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
-
+
key = post.get("key").trim();
value = post.get("value").trim();
if (key != null && !key.isEmpty()) {
diff --git a/htroot/ConfigRobotsTxt_p.java b/source/net/yacy/htroot/ConfigRobotsTxt_p.java
similarity index 98%
rename from htroot/ConfigRobotsTxt_p.java
rename to source/net/yacy/htroot/ConfigRobotsTxt_p.java
index bf140e8c2..eb559f6fd 100644
--- a/htroot/ConfigRobotsTxt_p.java
+++ b/source/net/yacy/htroot/ConfigRobotsTxt_p.java
@@ -29,6 +29,8 @@
// javac -classpath .:../classes ConfigRobotsTxt_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@@ -45,7 +47,7 @@ public class ConfigRobotsTxt_p {
final RobotsTxtConfig rbc = sb.robotstxtConfig;
// globaly overwitten by template engine (removed to avoid risk of NPE in intranet mode)
- // prop.put("clientname", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()));
+ // prop.put("clientname", sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()));
if (post != null) {
if (post.containsKey("save")) {
diff --git a/htroot/ConfigSearchBox.java b/source/net/yacy/htroot/ConfigSearchBox.java
similarity index 95%
rename from htroot/ConfigSearchBox.java
rename to source/net/yacy/htroot/ConfigSearchBox.java
index 05bde9ac3..8c125b6db 100644
--- a/htroot/ConfigSearchBox.java
+++ b/source/net/yacy/htroot/ConfigSearchBox.java
@@ -23,6 +23,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.http.servlets.YaCyDefaultServlet;
import net.yacy.search.Switchboard;
diff --git a/htroot/ConfigSearchPage_p.java b/source/net/yacy/htroot/ConfigSearchPage_p.java
similarity index 94%
rename from htroot/ConfigSearchPage_p.java
rename to source/net/yacy/htroot/ConfigSearchPage_p.java
index 09967e60f..adebd8cf9 100644
--- a/htroot/ConfigSearchPage_p.java
+++ b/source/net/yacy/htroot/ConfigSearchPage_p.java
@@ -25,6 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.sql.Date;
import java.util.HashSet;
import java.util.Map;
@@ -56,21 +58,21 @@ public class ConfigSearchPage_p {
if (post != null) {
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(header, post);
-
+
final int initialNavMaxCOunt = sb.getConfigInt(
SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT);
final String initialNavConf = sb.getConfig("search.navigation", "");
-
+
if (post.containsKey("searchpage_set")) {
final String newGreeting = post.get(SwitchboardConstants.GREETING, "");
// store this call as api call
sb.tables.recordAPICall(post, "ConfigPortal_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);
sb.setConfig("publicTopmenu", post.getBoolean("publicTopmenu"));
-
+
sb.setConfig(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN,
post.getBoolean(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN));
-
+
sb.setConfig("search.options", post.getBoolean("search.options"));
sb.setConfig("search.text", post.getBoolean("search.text"));
@@ -81,14 +83,14 @@ public class ConfigSearchPage_p {
sb.setConfig(SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON, post.getBoolean(SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON));
sb.setConfig(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS, post.getBoolean(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS));
-
+
// maximum number of initially displayed keywords/tags
- int keywordsFirstMaxCount = post.getInt(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT,
+ final int keywordsFirstMaxCount = post.getInt(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT,
SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT_DEFAULT);
if (keywordsFirstMaxCount > 0) {
sb.setConfig(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT, keywordsFirstMaxCount);
}
-
+
sb.setConfig("search.result.show.date", post.getBoolean("search.result.show.date"));
sb.setConfig("search.result.show.size", post.getBoolean("search.result.show.size"));
sb.setConfig("search.result.show.metadata", post.getBoolean("search.result.show.metadata"));
@@ -101,7 +103,7 @@ public class ConfigSearchPage_p {
sb.setConfig("search.result.show.snapshots", post.getBoolean("search.result.show.snapshots"));
// construct navigation String
- Set<String> navConfigs = new HashSet<>();
+ final Set<String> navConfigs = new HashSet<>();
if (post.getBoolean("search.navigation.location")) {
navConfigs.add("location");
}
@@ -115,7 +117,7 @@ public class ConfigSearchPage_p {
navConfigs.add("date");
}
// append active navigator plugins
- String[] activeNavNames = post.getAll("search.navigation.active");
+ final String[] activeNavNames = post.getAll("search.navigation.active");
for (final String navName : activeNavNames) {
String navConfig = navName;
final String navSortConfig = post.get("search.navigation." + navName + ".navSort");
@@ -129,13 +131,13 @@ public class ConfigSearchPage_p {
}
sb.setConfig("search.navigation", navConfigs);
// maxcount nav entries, default
- int navmaxcnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT);
+ final int navmaxcnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT);
if (navmaxcnt > 5) {
sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, navmaxcnt);
}
-
+
// maxcount dates navigator entries
- int datesNavMaxCnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
+ final int datesNavMaxCnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT);
if (datesNavMaxCnt > 5) {
sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, datesNavMaxCnt);
@@ -143,7 +145,7 @@ public class ConfigSearchPage_p {
}
if (post.containsKey("add.nav")) { // button: add navigator plugin to ative list
- String navname = post.get("search.navigation.navname");
+ final String navname = post.get("search.navigation.navname");
if (navname != null && !navname.isEmpty()) {
final Set<String> navConfigs = sb.getConfigSet("search.navigation");
navConfigs.add(navname);
@@ -196,14 +198,14 @@ public class ConfigSearchPage_p {
config.getProperty(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
String.valueOf(QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT)));
}
-
+
if(!initialNavConf.equals(sb.getConfig("search.navigation", "")) || initialNavMaxCOunt != sb.getConfigInt(
SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT)) {
/* Clean up search events cache when necessary */
SearchEventCache.cleanupEvents(true);
}
}
-
+
/* Acquire a transaction token for the next POST form submission */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));
@@ -214,11 +216,11 @@ public class ConfigSearchPage_p {
prop.putHTML(SwitchboardConstants.GREETING_IMAGE_ALT, sb.getConfig(SwitchboardConstants.GREETING_IMAGE_ALT, ""));
prop.putHTML(SwitchboardConstants.INDEX_FORWARD, sb.getConfig(SwitchboardConstants.INDEX_FORWARD, ""));
prop.put("publicTopmenu", sb.getConfigBool("publicTopmenu", false) ? 1 : 0);
-
+
prop.put(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN,
sb.getConfigBool(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN,
SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN_DEFAULT) ? 1 : 0);
-
+
prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0);
prop.put("search.text", sb.getConfigBool("search.text", false) ? 1 : 0);
@@ -226,7 +228,7 @@ public class ConfigSearchPage_p {
prop.put("search.audio", sb.getConfigBool("search.audio", false) ? 1 : 0);
prop.put("search.video", sb.getConfigBool("search.video", false) ? 1 : 0);
prop.put("search.app", sb.getConfigBool("search.app", false) ? 1 : 0);
-
+
prop.put(SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON,
sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON,
SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON_DEFAULT));
@@ -234,11 +236,11 @@ public class ConfigSearchPage_p {
prop.put(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS,
sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS,
SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS_DEFAULT) ? 1 : 0);
-
+
prop.put(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT,
sb.getConfigInt(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT,
SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT_DEFAULT));
-
+
prop.put("search.result.show.date", sb.getConfigBool("search.result.show.date", false) ? 1 : 0);
prop.put("search.result.show.size", sb.getConfigBool("search.result.show.size", false) ? 1 : 0);
prop.put("search.result.show.metadata", sb.getConfigBool("search.result.show.metadata", false) ? 1 : 0);
@@ -274,10 +276,10 @@ public class ConfigSearchPage_p {
prop.put("search.navigation.topics", topicsNavEnabled);
prop.put("search.navigation.date", dateNavEnabled);
// list active navigator plugins
- Map<String, Navigator> navplugins = NavigatorPlugins.initFromCfgStrings(navConfigs);
+ final Map<String, Navigator> navplugins = NavigatorPlugins.initFromCfgStrings(navConfigs);
int i = 0;
for (final String navname : navplugins.keySet()) {
- Navigator nav = navplugins.get(navname);
+ final Navigator nav = navplugins.get(navname);
prop.put("search.navigation.plugin_" + i + "_name", navname);
prop.put("search.navigation.plugin_" + i + "_displayname", nav.getDisplayName());
final int navSort;
@@ -286,16 +288,16 @@ public class ConfigSearchPage_p {
} else {
if(nav.getSort().getSortType() == NavigatorSortType.COUNT) {
if(nav.getSort().getSortDir() == NavigatorSortDirection.DESC) {
- navSort = 0;
+ navSort = 0;
} else {
navSort = 1;
}
} else {
if(nav.getSort().getSortDir() == NavigatorSortDirection.DESC) {
- navSort = 2;
+ navSort = 2;
} else {
navSort = 3;
- }
+ }
}
}
prop.put("search.navigation.plugin_" + i + "_navSort", navSort);
@@ -304,9 +306,9 @@ public class ConfigSearchPage_p {
prop.put("search.navigation.plugin", i);
// fill select field options (only navs not already active)
- Map<String, String> defaultnavplugins = NavigatorPlugins.listAvailable();
+ final Map<String, String> defaultnavplugins = NavigatorPlugins.listAvailable();
i=0;
- for (String navname : defaultnavplugins.keySet()) {
+ for (final String navname : defaultnavplugins.keySet()) {
if (!navplugins.containsKey(navname)) {
prop.put("search.navigation.list_" + i + "_name", navname);
prop.put("search.navigation.list_" + i + "_displayname", defaultnavplugins.get(navname));
@@ -322,7 +324,7 @@ public class ConfigSearchPage_p {
prop.put(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, sb.getConfigInt(
SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
-
+
prop.put(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, sb.getConfigInt(
SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
diff --git a/htroot/ConfigUpdate_p.java b/source/net/yacy/htroot/ConfigUpdate_p.java
similarity index 99%
rename from htroot/ConfigUpdate_p.java
rename to source/net/yacy/htroot/ConfigUpdate_p.java
index 9bc35e483..506d9e7f0 100644
--- a/htroot/ConfigUpdate_p.java
+++ b/source/net/yacy/htroot/ConfigUpdate_p.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -87,7 +89,7 @@ public class ConfigUpdate_p {
break;
}
}
- File downloadedRelease = versionToDownload.downloadRelease();
+ final File downloadedRelease = versionToDownload.downloadRelease();
if(downloadedRelease == null) {
prop.put("candeploy_downloadError", "1");
prop.putHTML("candeploy_downloadError_releasedownload", release);
diff --git a/htroot/ConfigUser_p.java b/source/net/yacy/htroot/ConfigUser_p.java
similarity index 93%
rename from htroot/ConfigUser_p.java
rename to source/net/yacy/htroot/ConfigUser_p.java
index a3c4a950b..b567b5c6b 100644
--- a/htroot/ConfigUser_p.java
+++ b/source/net/yacy/htroot/ConfigUser_p.java
@@ -18,6 +18,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.EnumMap;
import java.util.Map;
@@ -72,7 +74,7 @@ public class ConfigUser_p {
//do nothing
} else if (post.containsKey("user") && !"newuser".equals(post.get("user"))) {
- UserDB.Entry entry = sb.userDB.getEntry(post.get("user"));
+ final UserDB.Entry entry = sb.userDB.getEntry(post.get("user"));
if (entry != null) {
//TODO: set username read-only in html
prop.putHTML("current_user", entry.getUserName());
@@ -117,7 +119,7 @@ public class ConfigUser_p {
rightsSet.put(right, post.containsKey(right.toString()) && "on".equals(post.get(right.toString())) ? "true" : "false");
}
- UserDB.Entry entry = sb.userDB.getEntry(username);
+ final UserDB.Entry entry = sb.userDB.getEntry(username);
if (entry != null) {
try {
@@ -151,7 +153,7 @@ public class ConfigUser_p {
} catch (final Exception e) {
ConcurrentLog.logException(e);
}
- YaCyHttpServer jhttpserver = (YaCyHttpServer) sb.getHttpServer();
+ final YaCyHttpServer jhttpserver = (YaCyHttpServer) sb.getHttpServer();
jhttpserver.resetUser(entry.getUserName());
} else {
prop.put("error", "1");
@@ -162,7 +164,7 @@ public class ConfigUser_p {
prop.putHTML("username", username);
} else if (post.containsKey("delete")) {
sb.userDB.removeEntry(post.get("username"));
- YaCyHttpServer jhttpserver = (YaCyHttpServer) sb.getHttpServer();
+ final YaCyHttpServer jhttpserver = (YaCyHttpServer) sb.getHttpServer();
jhttpserver.removeUser(post.get("username"));
prop.put(serverObjects.ACTION_LOCATION, "ConfigAccountList_p.html"); // jump back to user list
}
diff --git a/htroot/Connections_p.java b/source/net/yacy/htroot/Connections_p.java
similarity index 96%
rename from htroot/Connections_p.java
rename to source/net/yacy/htroot/Connections_p.java
index 5d28da486..759c2ba24 100644
--- a/htroot/Connections_p.java
+++ b/source/net/yacy/htroot/Connections_p.java
@@ -27,6 +27,8 @@
//javac -classpath .:../classes Network.java
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
diff --git a/htroot/ContentAnalysis_p.java b/source/net/yacy/htroot/ContentAnalysis_p.java
similarity index 99%
rename from htroot/ContentAnalysis_p.java
rename to source/net/yacy/htroot/ContentAnalysis_p.java
index eed8455e7..a6972cd37 100644
--- a/htroot/ContentAnalysis_p.java
+++ b/source/net/yacy/htroot/ContentAnalysis_p.java
@@ -18,6 +18,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import net.yacy.cora.federate.solr.Ranking;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
diff --git a/htroot/ContentIntegrationPHPBB3_p.java b/source/net/yacy/htroot/ContentIntegrationPHPBB3_p.java
similarity index 82%
rename from htroot/ContentIntegrationPHPBB3_p.java
rename to source/net/yacy/htroot/ContentIntegrationPHPBB3_p.java
index 0f0cc441b..0c4c52cd2 100644
--- a/htroot/ContentIntegrationPHPBB3_p.java
+++ b/source/net/yacy/htroot/ContentIntegrationPHPBB3_p.java
@@ -22,6 +22,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.File;
import net.yacy.cora.date.GenericFormatter;
@@ -46,16 +48,16 @@ public class ContentIntegrationPHPBB3_p {
if (post != null) {
- String urlstub = post.get("content.phpbb3.urlstub", "");
- String dbtype = post.get("content.phpbb3.dbtype", "");
- String dbhost = post.get("content.phpbb3.dbhost", "");
- int dbport = post.getInt("content.phpbb3.dbport", 3306);
- String dbname = post.get("content.phpbb3.dbname", "");
- String prefix = post.get("content.phpbb3.tableprefix", "");
- String dbuser = post.get("content.phpbb3.dbuser", "");
- String dbpw = post.get("content.phpbb3.dbpw", "");
- int ppf = post.getInt("content.phpbb3.ppf", 1000);
- String dumpfile = post.get("content.phpbb3.dumpfile", "");
+ final String urlstub = post.get("content.phpbb3.urlstub", "");
+ final String dbtype = post.get("content.phpbb3.dbtype", "");
+ final String dbhost = post.get("content.phpbb3.dbhost", "");
+ final int dbport = post.getInt("content.phpbb3.dbport", 3306);
+ final String dbname = post.get("content.phpbb3.dbname", "");
+ final String prefix = post.get("content.phpbb3.tableprefix", "");
+ final String dbuser = post.get("content.phpbb3.dbuser", "");
+ final String dbpw = post.get("content.phpbb3.dbpw", "");
+ final int ppf = post.getInt("content.phpbb3.ppf", 1000);
+ final String dumpfile = post.get("content.phpbb3.dumpfile", "");
sb.setConfig("content.phpbb3.urlstub", urlstub);
@@ -71,7 +73,7 @@ public class ContentIntegrationPHPBB3_p {
if (post.containsKey("check")) {
try {
- Dao db = new PhpBB3Dao(
+ final Dao db = new PhpBB3Dao(
urlstub,
dbtype,
dbhost,
@@ -95,7 +97,7 @@ public class ContentIntegrationPHPBB3_p {
if (post.containsKey("export")) {
try {
- Dao db = new PhpBB3Dao(
+ final Dao db = new PhpBB3Dao(
urlstub,
dbtype,
dbhost,
@@ -106,7 +108,7 @@ public class ContentIntegrationPHPBB3_p {
dbpw
);
- int files = db.writeSurrogates(db.query(0, -1, 100), sb.surrogatesInPath, "fullexport-" + GenericFormatter.SHORT_SECOND_FORMATTER.format(), ppf);
+ final int files = db.writeSurrogates(db.query(0, -1, 100), sb.surrogatesInPath, "fullexport-" + GenericFormatter.SHORT_SECOND_FORMATTER.format(), ppf);
prop.put("export", 1);
prop.put("export_files", files);
db.close();
@@ -118,12 +120,12 @@ public class ContentIntegrationPHPBB3_p {
}
if (post.containsKey("import")) {
- File f = new File(dumpfile);
+ final File f = new File(dumpfile);
if (!f.exists()) {
prop.put("import", 2);
prop.put("import_error", "file " + dumpfile + " does not exist");
} else try {
- ImportDump importer = new ImportDump(
+ final ImportDump importer = new ImportDump(
dbtype,
dbhost,
dbport,
diff --git a/htroot/CookieMonitorIncoming_p.java b/source/net/yacy/htroot/CookieMonitorIncoming_p.java
similarity index 96%
rename from htroot/CookieMonitorIncoming_p.java
rename to source/net/yacy/htroot/CookieMonitorIncoming_p.java
index ebdbbd850..76923822e 100644
--- a/htroot/CookieMonitorIncoming_p.java
+++ b/source/net/yacy/htroot/CookieMonitorIncoming_p.java
@@ -27,6 +27,8 @@
// javac -classpath .:../classes Network.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
diff --git a/htroot/CookieMonitorOutgoing_p.java b/source/net/yacy/htroot/CookieMonitorOutgoing_p.java
similarity index 96%
rename from htroot/CookieMonitorOutgoing_p.java
rename to source/net/yacy/htroot/CookieMonitorOutgoing_p.java
index 23f77e217..d8e155468 100644
--- a/htroot/CookieMonitorOutgoing_p.java
+++ b/source/net/yacy/htroot/CookieMonitorOutgoing_p.java
@@ -27,9 +27,12 @@
// javac -classpath .:../classes Network.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.Date;
import java.util.Iterator;
import java.util.Map;
+
import javax.servlet.http.Cookie;
import net.yacy.cora.protocol.HeaderFramework;
diff --git a/htroot/CookieTest_p.java b/source/net/yacy/htroot/CookieTest_p.java
similarity index 92%
rename from htroot/CookieTest_p.java
rename to source/net/yacy/htroot/CookieTest_p.java
index 891692b4d..782fe8333 100644
--- a/htroot/CookieTest_p.java
+++ b/source/net/yacy/htroot/CookieTest_p.java
@@ -27,7 +27,10 @@
// javac -classpath .:../classes index.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import javax.servlet.http.Cookie;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.server.serverObjects;
@@ -50,7 +53,7 @@ public class CookieTest_p {
final servletProperties prop = new servletProperties();
if (post.containsKey("act") && "clear_cookie".equals(post.get("act"))) {
final ResponseHeader outgoingHeader = new ResponseHeader(200);
- Cookie[] cookies = header.getCookies();
+ final Cookie[] cookies = header.getCookies();
if (cookies != null) {
for (final Cookie cookie : cookies) {
outgoingHeader.setCookie(cookie.getName(), cookie.getValue(), cookie.getMaxAge(), cookie.getPath(), cookie.getDomain(), cookie.getSecure());
@@ -71,10 +74,10 @@ public class CookieTest_p {
prop.putHTML("cookiesin_0_value", cookieValue);
}
- Cookie[] cookielst = header.getCookies();
+ final Cookie[] cookielst = header.getCookies();
int i = 0;
if (cookielst != null) {
- for (Cookie singleco : cookielst) {
+ for (final Cookie singleco : cookielst) {
prop.putHTML("cookiesout_" + i + "_string", singleco.getName() + "=" + singleco.getValue() + ";"); // output with ";" for compatiblity with cookiesin
i++;
}
diff --git a/htroot/CrawlCheck_p.java b/source/net/yacy/htroot/CrawlCheck_p.java
similarity index 86%
rename from htroot/CrawlCheck_p.java
rename to source/net/yacy/htroot/CrawlCheck_p.java
index 29b51f836..8a55dd466 100644
--- a/htroot/CrawlCheck_p.java
+++ b/source/net/yacy/htroot/CrawlCheck_p.java
@@ -18,6 +18,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.net.MalformedURLException;
import java.util.Collection;
import java.util.LinkedHashSet;
@@ -41,23 +43,23 @@ public class CrawlCheck_p {
final serverObjects prop = new serverObjects();
prop.put("starturls", "");
if (post == null) return prop;
-
+
if (post.containsKey("crawlcheck")) {
-
+
// get the list of rootURls for this crawl start
- Set<DigestURL> rootURLs = new LinkedHashSet<DigestURL>();
- String crawlingStart0 = post.get("crawlingURLs","").trim();
- String[] rootURLs0 = crawlingStart0.indexOf('\n') > 0 || crawlingStart0.indexOf('\r') > 0 ? crawlingStart0.split("[\\r\\n]+") : crawlingStart0.split(Pattern.quote("|"));
+ final Set<DigestURL> rootURLs = new LinkedHashSet<DigestURL>();
+ final String crawlingStart0 = post.get("crawlingURLs","").trim();
+ final String[] rootURLs0 = crawlingStart0.indexOf('\n') > 0 || crawlingStart0.indexOf('\r') > 0 ? crawlingStart0.split("[\\r\\n]+") : crawlingStart0.split(Pattern.quote("|"));
for (String crawlingStart: rootURLs0) {
if (crawlingStart == null || crawlingStart.length() == 0) continue;
// add the prefix http:// if necessary
- int pos = crawlingStart.indexOf("://",0);
+ final int pos = crawlingStart.indexOf("://",0);
if (pos == -1) {
if (crawlingStart.startsWith("www")) crawlingStart = "http://" + crawlingStart;
if (crawlingStart.startsWith("ftp")) crawlingStart = "ftp://" + crawlingStart;
}
try {
- DigestURL crawlingStartURL = new DigestURL(crawlingStart);
+ final DigestURL crawlingStartURL = new DigestURL(crawlingStart);
rootURLs.add(crawlingStartURL);
} catch (final MalformedURLException e) {
ConcurrentLog.logException(e);
@@ -68,18 +70,18 @@ public class CrawlCheck_p {
prop.put("table", 0);
} else {
prop.put("table", 1);
-
+
// mass check
final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
- Collection<CheckEntry> out = sb.robots.massCrawlCheck(rootURLs, agent);
+ final Collection<CheckEntry> out = sb.robots.massCrawlCheck(rootURLs, agent);
// evaluate the result from the concurrent computation
-
+
// make a string that is used to fill the starturls field again
// and analyze the urls to make the table rows
- StringBuilder s = new StringBuilder(300);
+ final StringBuilder s = new StringBuilder(300);
int row = 0;
- for (CheckEntry entry: out) {
- String u = entry.digestURL.toNormalform(true);
+ for (final CheckEntry entry: out) {
+ final String u = entry.digestURL.toNormalform(true);
s.append(u).append('\n');
prop.put("table_list_" + row + "_url", u);
@@ -95,7 +97,7 @@ public class CrawlCheck_p {
prop.put("table_list_" + row + "_crawldelay", Math.max(agent.minimumDelta, entry.robotsTxtEntry.getCrawlDelayMillis()) + " ms");
prop.put("table_list_" + row + "_sitemap", entry.robotsTxtEntry.getSitemaps().toString());
}
-
+
// try to load the url
if (robotsAllowed) {
if (entry.response == null) {
@@ -111,14 +113,14 @@ public class CrawlCheck_p {
prop.put("table_list_" + row + "_access", "not loaded - prevented by robots.txt");
}
row++;
-
+
}
prop.put("table_list", row);
prop.put("starturls", s.toString());
-
+
}
}
-
+
return prop;
}
diff --git a/htroot/CrawlMonitorRemoteStart.java b/source/net/yacy/htroot/CrawlMonitorRemoteStart.java
similarity index 98%
rename from htroot/CrawlMonitorRemoteStart.java
rename to source/net/yacy/htroot/CrawlMonitorRemoteStart.java
index 906de0ea5..f41802ba9 100644
--- a/htroot/CrawlMonitorRemoteStart.java
+++ b/source/net/yacy/htroot/CrawlMonitorRemoteStart.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Iterator;
import net.yacy.cora.protocol.RequestHeader;
@@ -41,7 +43,7 @@ public class CrawlMonitorRemoteStart {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
- boolean dark = true;
+ final boolean dark = true;
// create other peer crawl table using YaCyNews
Iterator<NewsDB.Record> recordIterator = sb.peers.newsPool.recordIterator(NewsPool.INCOMING_DB);
diff --git a/htroot/CrawlProfileEditor_p.java b/source/net/yacy/htroot/CrawlProfileEditor_p.java
similarity index 96%
rename from htroot/CrawlProfileEditor_p.java
rename to source/net/yacy/htroot/CrawlProfileEditor_p.java
index 4ca089ad2..5eee8b7c3 100644
--- a/htroot/CrawlProfileEditor_p.java
+++ b/source/net/yacy/htroot/CrawlProfileEditor_p.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Map;
import java.util.TreeMap;
@@ -100,7 +102,7 @@ public class CrawlProfileEditor_p {
if ((post != null) && (selentry != null)) {
if (post.containsKey("submit")) {
try {
- for (CrawlProfile.CrawlAttribute attribute: CrawlProfile.CrawlAttribute.values()) {
+ for (final CrawlProfile.CrawlAttribute attribute: CrawlProfile.CrawlAttribute.values()) {
final String cval = selentry.get(attribute.key);
final String val = (attribute.type == CrawlProfile.CrawlAttribute.BOOLEAN) ? Boolean.toString(post.containsKey(attribute.key)) : post.get(attribute.key, cval);
if (!cval.equals(val)) {
@@ -149,7 +151,7 @@ public class CrawlProfileEditor_p {
prop.put("edit_name", selentry.collectionName());
prop.put("edit_handle", selentry.handle());
count = 0;
- for (CrawlProfile.CrawlAttribute attribute: CrawlProfile.CrawlAttribute.values()) {
+ for (final CrawlProfile.CrawlAttribute attribute: CrawlProfile.CrawlAttribute.values()) {
final String val = selentry.get(attribute.key);
prop.put(EDIT_ENTRIES_PREFIX + count + "_readonly", attribute.readonly ? "1" : "0");
prop.put(EDIT_ENTRIES_PREFIX + count + "_readonly_name", attribute.key);
diff --git a/htroot/CrawlResults.java b/source/net/yacy/htroot/CrawlResults.java
similarity index 94%
rename from htroot/CrawlResults.java
rename to source/net/yacy/htroot/CrawlResults.java
index 262ed172f..2c10be06a 100644
--- a/htroot/CrawlResults.java
+++ b/source/net/yacy/htroot/CrawlResults.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
@@ -60,17 +62,17 @@ public class CrawlResults {
/** Used for logging. */
private static final String APP_NAME = "PLASMA";
-
+
/** Default maximum lines number of the indexed table */
private static final int DEFAULT_MAXIMUM_LINES = 500;
-
+
public static serverObjects respond(final RequestHeader header, serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
int lines = DEFAULT_MAXIMUM_LINES;
- boolean showCollection = sb.index.fulltext().getDefaultConfiguration().isEmpty() || sb.index.fulltext().getDefaultConfiguration().contains(CollectionSchema.collection_sxt);
+ final boolean showCollection = sb.index.fulltext().getDefaultConfiguration().isEmpty() || sb.index.fulltext().getDefaultConfiguration().contains(CollectionSchema.collection_sxt);
boolean showInit = env.getConfigBool("IndexMonitorInit", false);
boolean showExec = env.getConfigBool("IndexMonitorExec", false);
boolean showDate = env.getConfigBool("IndexMonitorDate", true);
@@ -119,7 +121,7 @@ public class CrawlResults {
return prop;
}
}
-
+
String selectedblacklist = post.get("selectedblacklist",Blacklist.defaultBlacklist(ListManager.listsPath));
if (post != null) {
// custom number of lines
@@ -142,18 +144,18 @@ public class CrawlResults {
final String domain = post.get("domain", null);
if (domain != null) {
selectedblacklist = post.get("blacklistname");
- Set<String> hostnames = new HashSet<String>();
+ final Set<String> hostnames = new HashSet<String>();
hostnames.add(domain);
sb.index.fulltext().deleteStaleDomainNames(hostnames, null);
ResultURLs.deleteDomain(tabletype, domain);
-
+
// handle addtoblacklist
if (post.containsKey("delandaddtoblacklist")) {
try {
Switchboard.urlBlacklist.add(selectedblacklist, domain, ".*");
- } catch (PunycodeException e) {
+ } catch (final PunycodeException e) {
ConcurrentLog.warn(APP_NAME, "Unable to add blacklist entry to blacklist " + selectedblacklist, e);
-
+
}
}
}
@@ -213,7 +215,7 @@ public class CrawlResults {
while (i.hasNext() && cnt < lines) {
entry = i.next();
try {
- byte[] urlhash = UTF8.getBytes(entry.getKey());
+ final byte[] urlhash = UTF8.getBytes(entry.getKey());
urle = sb.index.fulltext().getMetadata(urlhash);
if (urle == null) {
sb.index.fulltext().commit(true);
@@ -331,7 +333,7 @@ public class CrawlResults {
prop.put("table_domains", cnt);
// load all blacklist files located in the directory
- List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
+ final List<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
int blacklistCount = 0;
if (dirlist != null) {
for (final String element : dirlist) {
@@ -345,7 +347,7 @@ public class CrawlResults {
blacklistCount++;
}
prop.put("table_blacklists", blacklistCount);
- }
+ }
}
prop.put("process", tabletype.getCode());
@@ -356,7 +358,7 @@ public class CrawlResults {
private static final DateTimeFormatter DAY_FORMATTER = DateTimeFormatter
.ofPattern("uuuu/MM/dd", Locale.US).withZone(ZoneId.systemDefault());
-
+
/**
* @param date a date to render as a String
* @return the date formatted using the DAY_FORMATTER pattern.
diff --git a/htroot/CrawlStartExpert.java b/source/net/yacy/htroot/CrawlStartExpert.java
similarity index 97%
rename from htroot/CrawlStartExpert.java
rename to source/net/yacy/htroot/CrawlStartExpert.java
index 635b76177..8a90510f5 100644
--- a/htroot/CrawlStartExpert.java
+++ b/source/net/yacy/htroot/CrawlStartExpert.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -172,7 +174,7 @@ public class CrawlStartExpert {
// Unlimited crawl depth for URLs matching with
if (post != null && post.containsKey("crawlingDepthExtension")) {
- prop.put("crawlingDepthExtension",
+ prop.put("crawlingDepthExtension",
post.get("crawlingDepthExtension", ""));
} else {
prop.put("crawlingDepthExtension", CrawlProfile.MATCH_NEVER_STRING);
@@ -314,7 +316,7 @@ public class CrawlStartExpert {
// Filter on Content of Document: must match
if (post != null && post.containsKey("indexcontentmustmatch")) {
- prop.put("indexcontentmustmatch",
+ prop.put("indexcontentmustmatch",
post.get("indexcontentmustmatch", ""));
} else {
prop.put("indexcontentmustmatch", CrawlProfile.MATCH_ALL_STRING);
@@ -562,7 +564,7 @@ public class CrawlStartExpert {
}
// ---------- Agent name
- List<String> agentNames = new ArrayList<String>();
+ final List<String> agentNames = new ArrayList<String>();
if (sb.isIntranetMode()) {
agentNames.add(ClientIdentification.yacyIntranetCrawlerAgentName);
}
@@ -576,7 +578,7 @@ public class CrawlStartExpert {
}
String defaultAgentName = agentNames.get(0);
if (post != null && post.containsKey("agentName")) {
- String agentName = post.get("agentName", sb.isIntranetMode() ? ClientIdentification.yacyIntranetCrawlerAgentName : ClientIdentification.yacyInternetCrawlerAgentName);
+ final String agentName = post.get("agentName", sb.isIntranetMode() ? ClientIdentification.yacyIntranetCrawlerAgentName : ClientIdentification.yacyInternetCrawlerAgentName);
if (agentNames.contains(agentName)) defaultAgentName = agentName;
}
for (int i = 0; i < agentNames.size(); i++) {
@@ -588,21 +590,21 @@ public class CrawlStartExpert {
// ---------- Ignore Class Name
if (post != null && post.containsKey("ignoreclassname")) {
- prop.put("ignoreclassname",
+ prop.put("ignoreclassname",
post.get("ignoreclassname", ""));
} else {
prop.put("ignoreclassname", "");
}
// ---------- Enrich Vocabulary
- Collection<Tagging> vocs = LibraryProvider.autotagging.getVocabularies();
+ final Collection<Tagging> vocs = LibraryProvider.autotagging.getVocabularies();
if (vocs.size() == 0) {
prop.put("vocabularySelect", 0);
} else {
prop.put("vocabularySelect", 1);
int count = 0;
- for (Tagging v: vocs) {
- String value = post == null ? "" : post.get("vocabulary_" + v.getName() + "_class", "");
+ for (final Tagging v: vocs) {
+ final String value = post == null ? "" : post.get("vocabulary_" + v.getName() + "_class", "");
prop.put("vocabularySelect_vocabularyset_" + count + "_name", v.getName());
prop.put("vocabularySelect_vocabularyset_" + count + "_value", value);
count++;
@@ -611,7 +613,7 @@ public class CrawlStartExpert {
}
// ---------- Snapshot generation
- boolean wkhtmltopdfAvailable = Html2Image.wkhtmltopdfAvailable();
+ final boolean wkhtmltopdfAvailable = Html2Image.wkhtmltopdfAvailable();
//boolean convertAvailable = Html2Image.convertAvailable();
prop.put("snapshotsMaxDepth", post == null ? "-1" : post.get("snapshotsMaxDepth", "-1"));
prop.put("snapshotsMustnotmatch", post == null ? "" : post.get("snapshotsMustnotmatch", ""));
@@ -660,7 +662,7 @@ public class CrawlStartExpert {
}
// Target collection
- boolean collectionEnabled =
+ final boolean collectionEnabled =
sb.index.fulltext().getDefaultConfiguration().isEmpty() ||
sb.index.fulltext().getDefaultConfiguration().contains(
CollectionSchema.collection_sxt);
diff --git a/htroot/CrawlStartScanner_p.java b/source/net/yacy/htroot/CrawlStartScanner_p.java
similarity index 93%
rename from htroot/CrawlStartScanner_p.java
rename to source/net/yacy/htroot/CrawlStartScanner_p.java
index ed4e050ba..d76cf08b3 100644
--- a/htroot/CrawlStartScanner_p.java
+++ b/source/net/yacy/htroot/CrawlStartScanner_p.java
@@ -18,6 +18,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.io.IOException;
import java.net.InetAddress;
import java.net.MalformedURLException;
@@ -79,9 +81,9 @@ public class CrawlStartScanner_p {
listall = true;
post.put("source", "hosts");
}
- String[] hosts0 = hostt.indexOf('\n') > 0 || hostt.indexOf('\r') > 0 ? hostt.split("[\\r\\n]+") : hostt.split(Pattern.quote(","));
- Set<String> hostSet = new LinkedHashSet<String>();
- for (String s: hosts0) if (s != null && s.length() > 0) hostSet.add(s);
+ final String[] hosts0 = hostt.indexOf('\n') > 0 || hostt.indexOf('\r') > 0 ? hostt.split("[\\r\\n]+") : hostt.split(Pattern.quote(","));
+ final Set<String> hostSet = new LinkedHashSet<String>();
+ for (final String s: hosts0) if (s != null && s.length() > 0) hostSet.add(s);
final Set<InetAddress> ips = Domains.myIntranetIPs();
prop.put("intranethosts", ips.toString());
@@ -116,7 +118,7 @@ public class CrawlStartScanner_p {
}
}
}
- String hos = ""; for (String s: hostSet) hos += s + "\n";
+ String hos = ""; for (final String s: hostSet) hos += s + "\n";
prop.put("scanhosts", hos.trim());
if (listall) {
@@ -126,7 +128,7 @@ public class CrawlStartScanner_p {
hostscore = sb.index.fulltext().getDefaultConnector().getFacets(AbstractSolrConnector.CATCHALL_QUERY, 1000, CollectionSchema.host_s.getSolrFieldName()).get(CollectionSchema.host_s.getSolrFieldName());
} catch (final IOException e) {}
if (hostscore != null) {
- for (String s: hostscore) hostSet.add(s);
+ for (final String s: hostscore) hostSet.add(s);
}
}
@@ -149,12 +151,12 @@ public class CrawlStartScanner_p {
// start a scanner
final Scanner scanner = new Scanner(CONCURRENT_RUNNER, timeout);
- boolean scanhttp = "on".equals(post.get("scanhttp", ""));
- int scanhttpport = post.getInt("scanhttpport", 80);
- boolean scanhttps = "on".equals(post.get("scanhttps", ""));
- int scanhttpsport = post.getInt("scanhttpsport", 443);
- boolean scanftp = "on".equals(post.get("scanftp", ""));
- boolean scansmb = "on".equals(post.get("scansmb", ""));
+ final boolean scanhttp = "on".equals(post.get("scanhttp", ""));
+ final int scanhttpport = post.getInt("scanhttpport", 80);
+ final boolean scanhttps = "on".equals(post.get("scanhttps", ""));
+ final int scanhttpsport = post.getInt("scanhttpsport", 443);
+ final boolean scanftp = "on".equals(post.get("scanftp", ""));
+ final boolean scansmb = "on".equals(post.get("scansmb", ""));
// select host base to scan
if ("hosts".equals(post.get("source", ""))) {
@@ -166,7 +168,7 @@ public class CrawlStartScanner_p {
final int p = host.indexOf('/', 0);
if (p >= 0) host = host.substring(0, p);
InetAddress ip;
- Collection<InetAddress> scanbase = new ArrayList<InetAddress>();
+ final Collection<InetAddress> scanbase = new ArrayList<InetAddress>();
if (host.length() > 0) {
ip = Domains.dnsResolve(host); if (ip != null) scanbase.add(ip);
if (host.startsWith("ftp.") || host.startsWith("www.") ||
diff --git a/htroot/CrawlStartSite.java b/source/net/yacy/htroot/CrawlStartSite.java
similarity index 95%
rename from htroot/CrawlStartSite.java
rename to source/net/yacy/htroot/CrawlStartSite.java
index 4152b3f04..b7b74cd26 100644
--- a/htroot/CrawlStartSite.java
+++ b/source/net/yacy/htroot/CrawlStartSite.java
@@ -9,7 +9,7 @@
// $LastChangedBy$
//
// LICENSE
-//
+//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@@ -24,12 +24,14 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
public class CrawlStartSite {
-
+
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
return CrawlStartExpert.respond(header, post, env);
}
diff --git a/htroot/Crawler_p.java b/source/net/yacy/htroot/Crawler_p.java
similarity index 99%
rename from htroot/Crawler_p.java
rename to source/net/yacy/htroot/Crawler_p.java
index 740d057c1..490f963b7 100644
--- a/htroot/Crawler_p.java
+++ b/source/net/yacy/htroot/Crawler_p.java
@@ -19,6 +19,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@@ -416,7 +418,7 @@ public class Crawler_p {
if ("smb.ftp".indexOf(u.getProtocol()) >= 0 || "pdf".equals(MultiProtocolURL.getFileExtension(u.getFileName()))) anysmbftporpdf = true;
}
sb.index.fulltext().remove(deleteIDs);
- deleteIDs.forEach(urlhash -> {try {sb.index.loadTimeIndex().remove(urlhash.getBytes());} catch (IOException e) {}});
+ deleteIDs.forEach(urlhash -> {try {sb.index.loadTimeIndex().remove(urlhash.getBytes());} catch (final IOException e) {}});
sb.crawlQueues.removeHosts(hosthashes);
sb.index.fulltext().commit(true);
@@ -438,7 +440,7 @@ public class Crawler_p {
String basepath = u.toNormalform(true);
if (!basepath.endsWith("/")) {final int p = basepath.lastIndexOf("/"); if (p > 0) basepath = basepath.substring(0, p + 1);}
final int count = sb.index.fulltext().remove(basepath, deleteageDate);
- try {sb.index.loadTimeIndex().clear();} catch (IOException e) {}
+ try {sb.index.loadTimeIndex().clear();} catch (final IOException e) {}
if (count > 0) ConcurrentLog.info("Crawler_p", "deleted " + count + " documents for host " + u.getHost());
}
}
@@ -868,7 +870,7 @@ public class Crawler_p {
* @throws IOException
* @throws FileNotFoundException
*/
- private static List<AnchorURL> crawlingFileStart(final File crawlingFile, int timezoneOffset,
+ private static List<AnchorURL> crawlingFileStart(final File crawlingFile, final int timezoneOffset,
final String crawlingFileContent) throws MalformedURLException, IOException, FileNotFoundException {
List<AnchorURL> hyperlinks_from_file;
// check if the crawl filter works correctly
diff --git a/htroot/DictionaryLoader_p.java b/source/net/yacy/htroot/DictionaryLoader_p.java
similarity index 98%
rename from htroot/DictionaryLoader_p.java
rename to source/net/yacy/htroot/DictionaryLoader_p.java
index 09098a7df..9b415d46b 100644
--- a/htroot/DictionaryLoader_p.java
+++ b/source/net/yacy/htroot/DictionaryLoader_p.java
@@ -18,6 +18,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -246,7 +248,7 @@ public class DictionaryLoader_p {
prop.put("geo2ActionLoaded_error", e.getMessage());
}
}
-
+
if (post.containsKey("geo0Remove")) {
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.file());
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.fileDisabled());
@@ -259,7 +261,7 @@ public class DictionaryLoader_p {
LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB0.nickname);
prop.put("geo0ActionDeactivated", 1);
}
-
+
if (post.containsKey("geo1Remove")) {
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.file());
FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.fileDisabled());
@@ -336,7 +338,7 @@ public class DictionaryLoader_p {
LibraryProvider.initDidYouMean();
prop.put("drw0ActionActivated", 1);
}
-
+
if (post.containsKey("syn0Deactivate")) {
synonym_de_production.delete();
SynonymLibrary.init(synonyms_path);
@@ -347,29 +349,29 @@ public class DictionaryLoader_p {
try {
inStream = new FileInputStream(synonym_de_default);
FileUtils.copy(inStream, synonym_de_production);
- } catch (IOException e) {
+ } catch (final IOException e) {
ConcurrentLog.logException(e);
} finally {
if(inStream != null) {
try {
inStream.close();
- } catch (IOException ignored) {
+ } catch (final IOException ignored) {
ConcurrentLog.warn("DictionaryLoader", "Could not close file " + synonym_de_default.getName());
}
}
}
SynonymLibrary.init(synonyms_path);
}
-
+
if (post.containsKey("syn1Deactivate")) {
synonym_en_production.delete();
SynonymLibrary.init(synonyms_path);
}
-
+
if (post.containsKey("syn1Activate")) {
try {
FileUtils.copy(new FileInputStream(synonym_en_default), synonym_en_production);
- } catch (IOException e) {
+ } catch (final IOException e) {
ConcurrentLog.logException(e);
}
SynonymLibrary.init(synonyms_path);
@@ -383,22 +385,22 @@ public class DictionaryLoader_p {
if (post.containsKey("syn2Activate")) {
try {
FileUtils.copy(new FileInputStream(synonym_ru_default), synonym_ru_production);
- } catch (IOException e) {
+ } catch (final IOException e) {
ConcurrentLog.logException(e);
}
SynonymLibrary.init(synonyms_path);
- }
+ }
if (post != null) {
// check here if there is a possibility synonym libraries have been activated/deactivated
prop.put("syn0Status", synonym_de_production.exists() ? 1 : 0);
prop.put("syn1Status", synonym_en_production.exists() ? 1 : 0);
prop.put("syn2Status", synonym_ru_production.exists() ? 1 : 0);
}
-
+
// check status again
boolean keepPlacesTagging = false;
for (final LibraryProvider.Dictionary dictionary: LibraryProvider.Dictionary.values()) {
- int newstatus = dictionary.file().exists() ? 1 : dictionary.fileDisabled().exists() ? 2 : 0;
+ final int newstatus = dictionary.file().exists() ? 1 : dictionary.fileDisabled().exists() ? 2 : 0;
if (newstatus == 1) keepPlacesTagging = true;
prop.put(dictionary.nickname + "Status", newstatus);
}
@@ -407,7 +409,7 @@ public class DictionaryLoader_p {
if (!keepPlacesTagging) {
LibraryProvider.autotagging.removePlaces();
}
-
+
return prop; // return rewrite values for templates
}
}
diff --git a/htroot/Help.java b/source/net/yacy/htroot/Help.java
similarity index 92%
rename from htroot/Help.java
rename to source/net/yacy/htroot/Help.java
index 875f36d2c..81d2aa03a 100644
--- a/htroot/Help.java
+++ b/source/net/yacy/htroot/Help.java
@@ -4,6 +4,8 @@
//$LastChangedBy$
//
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
diff --git a/htroot/IndexBrowser_p.java b/source/net/yacy/htroot/IndexBrowser_p.java
similarity index 76%
rename from htroot/IndexBrowser_p.java
rename to source/net/yacy/htroot/IndexBrowser_p.java
index bf66476f0..0751912bd 100644
--- a/htroot/IndexBrowser_p.java
+++ b/source/net/yacy/htroot/IndexBrowser_p.java
@@ -18,6 +18,10 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
+import static net.yacy.repository.BlacklistHelper.addBlacklistEntry;
+
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
@@ -60,7 +64,6 @@ import net.yacy.crawler.retrieval.Request;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.peers.graphics.WebStructureGraph.StructureEntry;
import net.yacy.repository.Blacklist;
-import static net.yacy.repository.BlacklistHelper.addBlacklistEntry;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.index.Fulltext;
@@ -125,7 +128,7 @@ public class IndexBrowser_p {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
- Fulltext fulltext = sb.index.fulltext();
+ final Fulltext fulltext = sb.index.fulltext();
final boolean autoload = sb.getConfigBool("browser.autoload", true);
final boolean load4everyone = sb.getConfigBool("browser.load4everyone", false);
final boolean loadRight = autoload || load4everyone; // add config later
@@ -173,7 +176,7 @@ public class IndexBrowser_p {
if (sb.crawlStacker.size() > 2) wait = false;
try {
url = new DigestURL(load);
- String reasonString = sb.crawlStacker.stackCrawl(new Request(
+ final String reasonString = sb.crawlStacker.stackCrawl(new Request(
sb.peers.mySeed().hash.getBytes(),
url, null, load, new Date(),
sb.crawler.defaultProxyProfile.handle(),
@@ -209,10 +212,10 @@ public class IndexBrowser_p {
if (post.containsKey("hosts")) {
// generate host list
try {
- boolean onlyCrawling = "crawling".equals(post.get("hosts", ""));
- boolean onlyErrors = "error".equals(post.get("hosts", ""));
+ final boolean onlyCrawling = "crawling".equals(post.get("hosts", ""));
+ final boolean onlyErrors = "error".equals(post.get("hosts", ""));
- int maxcount = 2 * 3 * 2 * 5 * 7 * 2 * 3; // which makes nice matrixes for 2, 3, 4, 5, 6, 7, 8, 9 rows/colums
+ final int maxcount = 2 * 3 * 2 * 5 * 7 * 2 * 3; // which makes nice matrixes for 2, 3, 4, 5, 6, 7, 8, 9 rows/colums
// collect hosts from index
ReversibleScoreMap<String> hostscore = fulltext.getDefaultConnector().getFacets(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", maxcount, CollectionSchema.host_s.getSolrFieldName()).get(CollectionSchema.host_s.getSolrFieldName());
@@ -222,9 +225,9 @@ public class IndexBrowser_p {
final Map<String, Integer[]> crawler = sb.crawlQueues.noticeURL.getDomainStackHosts(StackType.LOCAL, sb.robots);
final Map<String, Integer> hostNameToPendingCount = new HashMap<>();
- for(Entry<String, Integer[]>crawlerEntry: crawler.entrySet()) {
+ for(final Entry<String, Integer[]>crawlerEntry: crawler.entrySet()) {
/* The local stack returns keys composed of "hostname:port" : we now sum pending URLs counts by host name */
- String hostName = Domains.stripToHostName(crawlerEntry.getKey());
+ final String hostName = Domains.stripToHostName(crawlerEntry.getKey());
Integer pendingCount = hostNameToPendingCount.get(hostName);
if(pendingCount == null) {
pendingCount = 0;
@@ -234,21 +237,21 @@ public class IndexBrowser_p {
}
// collect the errorurls
- Map<String, ReversibleScoreMap<String>> exclfacets = fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.excl.name(), maxcount, CollectionSchema.host_s.getSolrFieldName());
- ReversibleScoreMap<String> exclscore = exclfacets == null ? new ClusteredScoreMap<String>(true) : exclfacets.get(CollectionSchema.host_s.getSolrFieldName());
- Map<String, ReversibleScoreMap<String>> failfacets = fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.fail.name(), maxcount, CollectionSchema.host_s.getSolrFieldName());
- ReversibleScoreMap<String> failscore = failfacets == null ? new ClusteredScoreMap<String>(true) : failfacets.get(CollectionSchema.host_s.getSolrFieldName());
+ final Map<String, ReversibleScoreMap<String>> exclfacets = fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.excl.name(), maxcount, CollectionSchema.host_s.getSolrFieldName());
+ final ReversibleScoreMap<String> exclscore = exclfacets == null ? new ClusteredScoreMap<String>(true) : exclfacets.get(CollectionSchema.host_s.getSolrFieldName());
+ final Map<String, ReversibleScoreMap<String>> failfacets = fulltext.getDefaultConnector().getFacets(CollectionSchema.failtype_s.getSolrFieldName() + ":" + FailType.fail.name(), maxcount, CollectionSchema.host_s.getSolrFieldName());
+ final ReversibleScoreMap<String> failscore = failfacets == null ? new ClusteredScoreMap<String>(true) : failfacets.get(CollectionSchema.host_s.getSolrFieldName());
int c = 0;
- Iterator<String> i = hostscore.keys(false);
+ final Iterator<String> i = hostscore.keys(false);
String host;
while (i.hasNext() && c < maxcount) {
host = i.next();
prop.putHTML("hosts_list_" + c + "_host", host);
- boolean inCrawler = hostNameToPendingCount.containsKey(host);
- int exclcount = exclscore.get(host);
- int failcount = failscore.get(host);
- int errors = exclcount + failcount;
+ final boolean inCrawler = hostNameToPendingCount.containsKey(host);
+ final int exclcount = exclscore.get(host);
+ final int failcount = failscore.get(host);
+ final int errors = exclcount + failcount;
prop.put("hosts_list_" + c + "_count", hostscore.get(host));
prop.put("hosts_list_" + c + "_crawler", inCrawler ? 1 : 0);
if (inCrawler) {
@@ -284,33 +287,33 @@ public class IndexBrowser_p {
// write host analysis if path after host is empty
if (uri.getPath().length() <= 1 && host != null && host.length() > 0 && sb.getConfigBool("decoration.hostanalysis", false)) {
//how many documents per crawldepth_i; get crawldepth_i facet for host
- ArrayList<String> ff = new ArrayList<>();
- for (CollectionSchema csf: CollectionSchema.values()) {
+ final ArrayList<String> ff = new ArrayList<>();
+ for (final CollectionSchema csf: CollectionSchema.values()) {
if ((csf.getType() != SolrType.num_integer && csf.getType() != SolrType.num_long) || csf.isMultiValued()) continue;
- String facetfield = csf.getSolrFieldName();
+ final String facetfield = csf.getSolrFieldName();
if (!fulltext.getDefaultConfiguration().contains(facetfield)) continue;
ff.add(csf.getSolrFieldName());
}
// add also vocabulary counters
- Map<String, ReversibleScoreMap<String>> vocabularyFacet = sb.index.fulltext().getDefaultConnector().getFacets(CollectionSchema.vocabularies_sxt.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM, 100, CollectionSchema.vocabularies_sxt.getSolrFieldName());
+ final Map<String, ReversibleScoreMap<String>> vocabularyFacet = sb.index.fulltext().getDefaultConnector().getFacets(CollectionSchema.vocabularies_sxt.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM, 100, CollectionSchema.vocabularies_sxt.getSolrFieldName());
if (vocabularyFacet.size() > 0) {
- Collection<String> vocnames = vocabularyFacet.values().iterator().next().keyList(true);
- for (String vocname: vocnames) {
+ final Collection<String> vocnames = vocabularyFacet.values().iterator().next().keyList(true);
+ for (final String vocname: vocnames) {
ff.add(CollectionSchema.VOCABULARY_PREFIX + vocname + CollectionSchema.VOCABULARY_LOGCOUNT_SUFFIX);
ff.add(CollectionSchema.VOCABULARY_PREFIX + vocname + CollectionSchema.VOCABULARY_LOGCOUNTS_SUFFIX);
}
}
// list the facets
- String[] facetfields = ff.toArray(new String[ff.size()]);
- Map<String, ReversibleScoreMap<String>> facets = fulltext.getDefaultConnector().getFacets(CollectionSchema.host_s.getSolrFieldName() + ":\"" + host + "\"", 100, facetfields);
+ final String[] facetfields = ff.toArray(new String[ff.size()]);
+ final Map<String, ReversibleScoreMap<String>> facets = fulltext.getDefaultConnector().getFacets(CollectionSchema.host_s.getSolrFieldName() + ":\"" + host + "\"", 100, facetfields);
int fc = 0;
- for (Map.Entry<String, ReversibleScoreMap<String>> facetentry: facets.entrySet()) {
- ReversibleScoreMap<String> facetfieldmap = facetentry.getValue();
+ for (final Map.Entry<String, ReversibleScoreMap<String>> facetentry: facets.entrySet()) {
+ final ReversibleScoreMap<String> facetfieldmap = facetentry.getValue();
if (facetfieldmap.size() == 0) continue;
- TreeMap<Long, Integer> statMap = new TreeMap<>();
- for (String k: facetfieldmap) statMap.put(Long.parseLong(k), facetfieldmap.get(k));
+ final TreeMap<Long, Integer> statMap = new TreeMap<>();
+ for (final String k: facetfieldmap) statMap.put(Long.parseLong(k), facetfieldmap.get(k));
prop.put("hostanalysis_facets_" + fc + "_facetname", facetentry.getKey());
- int c = 0; for (Entry<Long, Integer> entry: statMap.entrySet()) {
+ int c = 0; for (final Entry<Long, Integer> entry: statMap.entrySet()) {
prop.put("hostanalysis_facets_" + fc + "_facet_" + c + "_key", entry.getKey());
prop.put("hostanalysis_facets_" + fc + "_facet_" + c + "_count", entry.getValue());
prop.put("hostanalysis_facets_" + fc + "_facet_" + c + "_a", "solr/collection1/select?q=host_s:" + host + " AND " + facetentry.getKey() + ":" + entry.getKey() + "&defType=edismax&start=0&rows=1000&fl=sku,crawldepth_i");
@@ -334,8 +337,8 @@ public class IndexBrowser_p {
// try to re-load all urls that have load errors and matches with this prefix.
reload404 = true;
}
- int facetcount = post.getInt("facetcount", 0);
- boolean complete = post.getBoolean("complete");
+ final int facetcount = post.getInt("facetcount", 0);
+ final boolean complete = post.getBoolean("complete");
if (complete) { // we want only root paths for complete lists
p = path.indexOf('/', 10);
if (p > 0) path = path.substring(0, p + 1);
@@ -352,11 +355,11 @@ public class IndexBrowser_p {
// generate file list from path
prop.putHTML("outbound_host", host);
prop.putHTML("inbound_host", host);
- String hosthash = uri.hosthash();
- String[] pathparts = uri.getPaths();
+ final String hosthash = uri.hosthash();
+ final String[] pathparts = uri.getPaths();
// get all files for a specific host from the index
- StringBuilder q = new StringBuilder();
+ final StringBuilder q = new StringBuilder();
if (host == null) {
if (path.startsWith("file://")) {
q.append(CollectionSchema.url_protocol_s.getSolrFieldName()).append(":file");
@@ -365,7 +368,7 @@ public class IndexBrowser_p {
q.append(CollectionSchema.host_s.getSolrFieldName()).append(":\"").append(host).append("\"");
}
if (pathparts.length > 0 && pathparts[0].length() > 0) {
- for (String pe: pathparts) {
+ for (final String pe: pathparts) {
if (pe.length() > 0) q.append(" AND ").append(CollectionSchema.url_paths_sxt.getSolrFieldName()).append(":\"").append(pe).append('\"');
}
} else {
@@ -373,10 +376,10 @@ public class IndexBrowser_p {
q.append(" AND ").append(CollectionSchema.url_paths_sxt.getSolrFieldName()).append(AbstractSolrConnector.CATCHALL_DTERM);
}
}
-
+
// if reload errors is requested, limit results to pages with some failreason set
if(reload404) q.append(" AND ").append(CollectionSchema.failtype_s.getSolrFieldName()).append(AbstractSolrConnector.CATCHALL_DTERM);
-
+
final int pageSize = 100;
final BlockingQueue<SolrDocument> docs = new ArrayBlockingQueue<>(pageSize);
final List<String> queries = new ArrayList<>();
@@ -397,26 +400,26 @@ public class IndexBrowser_p {
CollectionSchema.references_exthosts_i.getSolrFieldName()
));
solrQueryTask.start();
- Set<String> storedDocs = new HashSet<String>();
- Map<String, FailType> errorDocs = new HashMap<String, FailType>();
- Set<String> inboundLinks = new HashSet<String>();
- Map<String, ReversibleScoreMap<String>> outboundHosts = new HashMap<String, ReversibleScoreMap<String>>();
- Map<String, InfoCacheEntry> infoCache = new HashMap<String, InfoCacheEntry>();
+ final Set<String> storedDocs = new HashSet<String>();
+ final Map<String, FailType> errorDocs = new HashMap<String, FailType>();
+ final Set<String> inboundLinks = new HashSet<String>();
+ final Map<String, ReversibleScoreMap<String>> outboundHosts = new HashMap<String, ReversibleScoreMap<String>>();
+ final Map<String, InfoCacheEntry> infoCache = new HashMap<String, InfoCacheEntry>();
int hostsize = 0;
final List<String> deleteIDs = new ArrayList<String>();
final Collection<String> reloadURLs = new ArrayList<String>();
final Set<String> reloadURLCollection = new HashSet<String>();
- long timeoutList = System.currentTimeMillis() + TIMEOUT;
+ final long timeoutList = System.currentTimeMillis() + TIMEOUT;
long remainingTime = TIMEOUT;
- long timeoutReferences = System.currentTimeMillis() + TIMEOUT;
- ReferenceReportCache rrCache = sb.index.getReferenceReportCache();
+ final long timeoutReferences = System.currentTimeMillis() + TIMEOUT;
+ final ReferenceReportCache rrCache = sb.index.getReferenceReportCache();
try {
SolrDocument doc = docs.poll(remainingTime, TimeUnit.MILLISECONDS);
while (doc != AbstractSolrConnector.POISON_DOCUMENT && doc != null) {
String u = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
- String errortype = (String) doc.getFieldValue(CollectionSchema.failtype_s.getSolrFieldName());
- FailType error = errortype == null ? null : FailType.valueOf(errortype);
- String ids = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName());
+ final String errortype = (String) doc.getFieldValue(CollectionSchema.failtype_s.getSolrFieldName());
+ final FailType error = errortype == null ? null : FailType.valueOf(errortype);
+ final String ids = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName());
infoCache.put(ids, new InfoCacheEntry(sb.index.fulltext(), rrCache, doc, ids, System.currentTimeMillis() < timeoutReferences));
if (u.startsWith(path)) {
if (delete) {
@@ -424,7 +427,7 @@ public class IndexBrowser_p {
} else {
if (error == null) storedDocs.add(u); else {
if (reload404 && error == FailType.fail) {
- ArrayList<String> collections = (ArrayList<String>) doc.getFieldValue(CollectionSchema.collection_sxt.getSolrFieldName());
+ final ArrayList<String> collections = (ArrayList<String>) doc.getFieldValue(CollectionSchema.collection_sxt.getSolrFieldName());
if (collections != null) reloadURLCollection.addAll(collections);
reloadURLs.add(u);
}
@@ -451,7 +454,7 @@ public class IndexBrowser_p {
while (links.hasNext()) {
u = links.next();
try {
- MultiProtocolURL mu = new MultiProtocolURL(u);
+ final MultiProtocolURL mu = new MultiProtocolURL(u);
if (mu.getHost() != null) {
ReversibleScoreMap<String> lks = outboundHosts.get(mu.getHost());
if (lks == null) {
@@ -477,47 +480,47 @@ public class IndexBrowser_p {
if (deleteIDs.size() > 0) sb.remove(deleteIDs);
if (reloadURLs.size() > 0) {
final Map<String, Pattern> cm = new LinkedHashMap<String, Pattern>();
- for (String collection: reloadURLCollection) cm.put(collection, QueryParams.catchall_pattern);
+ for (final String collection: reloadURLCollection) cm.put(collection, QueryParams.catchall_pattern);
sb.reload(reloadURLs, cm.size() > 0 ? cm : CrawlProfile.collectionParser("user"), false);
-
+
}
// collect from crawler
- List<Request> domainStackReferences = sb.crawlQueues.noticeURL.getDomainStackReferences(StackType.LOCAL, host, 1000, 3000);
- Set<String> loadingLinks = new HashSet<String>();
- for (Request crawlEntry: domainStackReferences) loadingLinks.add(crawlEntry.url().toNormalform(true));
+ final List<Request> domainStackReferences = sb.crawlQueues.noticeURL.getDomainStackReferences(StackType.LOCAL, host, 1000, 3000);
+ final Set<String> loadingLinks = new HashSet<String>();
+ for (final Request crawlEntry: domainStackReferences) loadingLinks.add(crawlEntry.url().toNormalform(true));
// now combine all lists into one
- Map<String, StoreType> files = new HashMap<String, StoreType>();
- for (String u: storedDocs) files.put(u, StoreType.INDEX);
- for (Map.Entry<String, FailType> e: errorDocs.entrySet()) files.put(e.getKey(), e.getValue() == FailType.fail ? StoreType.FAILED : StoreType.EXCLUDED);
- for (String u: inboundLinks) if (!files.containsKey(u)) files.put(u, StoreType.LINK);
- if(!reload404) for (String u: loadingLinks) if (u.startsWith(path) && !files.containsKey(u)) files.put(u, StoreType.LINK);
+ final Map<String, StoreType> files = new HashMap<String, StoreType>();
+ for (final String u: storedDocs) files.put(u, StoreType.INDEX);
+ for (final Map.Entry<String, FailType> e: errorDocs.entrySet()) files.put(e.getKey(), e.getValue() == FailType.fail ? StoreType.FAILED : StoreType.EXCLUDED);
+ for (final String u: inboundLinks) if (!files.containsKey(u)) files.put(u, StoreType.LINK);
+ if(!reload404) for (final String u: loadingLinks) if (u.startsWith(path) && !files.containsKey(u)) files.put(u, StoreType.LINK);
ConcurrentLog.info("IndexBrowser_p", "collected " + files.size() + " urls for path " + path);
// distinguish files and folders
- Map<String, Object> list = new TreeMap<String, Object>(); // a directory list; if object is boolean, its a file; if its a int[], then its a folder
- int pl = path.length();
+ final Map<String, Object> list = new TreeMap<String, Object>(); // a directory list; if object is boolean, its a file; if its a int[], then its a folder
+ final int pl = path.length();
String file;
- for (Map.Entry<String, StoreType> entry: files.entrySet()) {
+ for (final Map.Entry<String, StoreType> entry: files.entrySet()) {
if (entry.getKey().length() < pl) continue; // this is not inside the path
if (!entry.getKey().startsWith(path)) continue;
file = entry.getKey().substring(pl);
- StoreType type = entry.getValue();
+ final StoreType type = entry.getValue();
p = file.indexOf('/');
if (p < 0) {
// this is a file
list.put(entry.getKey(), type); // StoreType value: this is a file; true -> file is in index; false -> not in index, maybe in crawler
} else {
// this is a directory path or a file in a subdirectory
- String remainingPath = file.substring(0, p + 1);
+ final String remainingPath = file.substring(0, p + 1);
if (complete && remainingPath.indexOf('.') > 0) {
list.put(entry.getKey(), type); // StoreType value: this is a file
} else {
- String dir = path + remainingPath;
- Object c = list.get(dir);
+ final String dir = path + remainingPath;
+ final Object c = list.get(dir);
if (c == null) {
- int[] linkedStoredIncrawlerError = new int[]{0,0,0,0,0};
+ final int[] linkedStoredIncrawlerError = new int[]{0,0,0,0,0};
if (type == StoreType.LINK) linkedStoredIncrawlerError[0]++;
if (type == StoreType.INDEX) linkedStoredIncrawlerError[1]++;
if (loadingLinks.contains(entry.getKey())) linkedStoredIncrawlerError[2]++;
@@ -537,18 +540,18 @@ public class IndexBrowser_p {
int c = 0;
// first list only folders
int filecounter = 0;
- for (Map.Entry<String, Object> entry: list.entrySet()) {
+ for (final Map.Entry<String, Object> entry: list.entrySet()) {
if ((entry.getValue() instanceof StoreType)) {
filecounter++;
} else {
// this is a folder
prop.put("files_list_" + c + "_type", 1);
prop.putHTML("files_list_" + c + "_type_url", entry.getKey());
- int linked = ((int[]) entry.getValue())[0];
- int stored = ((int[]) entry.getValue())[1];
- int crawler = ((int[]) entry.getValue())[2];
- int excl = ((int[]) entry.getValue())[3];
- int error = ((int[]) entry.getValue())[4];
+ final int linked = ((int[]) entry.getValue())[0];
+ final int stored = ((int[]) entry.getValue())[1];
+ final int crawler = ((int[]) entry.getValue())[2];
+ final int excl = ((int[]) entry.getValue())[3];
+ final int error = ((int[]) entry.getValue())[4];
prop.put("files_list_" + c + "_type_stored", stored);
prop.put("files_list_" + c + "_type_linked", linked);
prop.put("files_list_" + c + "_type_pendingVisible", crawler > 0 ? 1 : 0);
@@ -561,34 +564,34 @@ public class IndexBrowser_p {
}
}
// then list files
- for (Map.Entry<String, Object> entry: list.entrySet()) {
+ for (final Map.Entry<String, Object> entry: list.entrySet()) {
if (entry.getValue() instanceof StoreType) {
// this is a file
prop.put("files_list_" + c + "_type", 0);
prop.putHTML("files_list_" + c + "_type_url", entry.getKey());
- StoreType type = (StoreType) entry.getValue();
+ final StoreType type = (StoreType) entry.getValue();
try {uri = new DigestURL(entry.getKey());} catch (final MalformedURLException e) {uri = null;}
- HarvestProcess process = uri == null ? null : sb.crawlQueues.exists(uri.hash()); // todo: cannot identify errors
- boolean loading = load.equals(entry.getKey()) || (process != null && process != HarvestProcess.ERRORS);
- boolean error = process == HarvestProcess.ERRORS || type == StoreType.EXCLUDED || type == StoreType.FAILED;
- boolean dc = type != StoreType.INDEX && !error && !loading && list.containsKey(entry.getKey() + "/");
+ final HarvestProcess process = uri == null ? null : sb.crawlQueues.exists(uri.hash()); // todo: cannot identify errors
+ final boolean loading = load.equals(entry.getKey()) || (process != null && process != HarvestProcess.ERRORS);
+ final boolean error = process == HarvestProcess.ERRORS || type == StoreType.EXCLUDED || type == StoreType.FAILED;
+ final boolean dc = type != StoreType.INDEX && !error && !loading && list.containsKey(entry.getKey() + "/");
if (!dc) {
prop.put("files_list_" + c + "_type_stored", type == StoreType.INDEX ? 1 : error ? 3 : loading ? 2 : 0 /*linked*/);
if (type == StoreType.INDEX) {
- String ids = ASCII.String(uri.hash());
- InfoCacheEntry ice = infoCache.get(ids);
+ final String ids = ASCII.String(uri.hash());
+ final InfoCacheEntry ice = infoCache.get(ids);
prop.put("files_list_" + c + "_type_stored_comment", ice == null ? "" : ice.toString()); // ice.toString() contains html, therefore do not use putHTML here
}
prop.put("files_list_" + c + "_type_stored_load", loadRight ? 1 : 0);
if (error) {
- FailType failType = errorDocs.get(entry.getKey());
+ final FailType failType = errorDocs.get(entry.getKey());
if (failType == null) {
// maybe this is only in the errorURL
//Metadata faildoc = sb.index.fulltext().getDefaultConnector().getMetadata(ASCII.String(uri.hash()));
prop.putHTML("files_list_" + c + "_type_stored_error", "unknown error");
} else {
- String ids = ASCII.String(uri.hash());
- InfoCacheEntry ice = infoCache.get(ids);
+ final String ids = ASCII.String(uri.hash());
+ final InfoCacheEntry ice = infoCache.get(ids);
prop.put("files_list_" + c + "_type_stored_error", failType == FailType.excl ? "excluded from indexing" : "load fail" + (ice == null ? "" : "; " + ice.toString()));
}
}
@@ -614,13 +617,13 @@ public class IndexBrowser_p {
prop.put("files_linkgraph_host", uri.getHost());
// generate inbound-links table
- StructureEntry struct = sb.webStructure.incomingReferences(hosthash);
+ final StructureEntry struct = sb.webStructure.incomingReferences(hosthash);
if (struct != null && struct.references.size() > 0) {
maxcount = 200;
- ReversibleScoreMap<String> score = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);
- for (Map.Entry<String, Integer> entry: struct.references.entrySet()) score.set(entry.getKey(), entry.getValue());
+ final ReversibleScoreMap<String> score = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);
+ for (final Map.Entry<String, Integer> entry: struct.references.entrySet()) score.set(entry.getKey(), entry.getValue());
c = 0;
- Iterator<String> i = score.keys(false);
+ final Iterator<String> i = score.keys(false);
while (i.hasNext() && c < maxcount) {
host = i.next();
prop.putHTML("inbound_list_" + c + "_host", sb.webStructure.hostHash2hostName(host));
@@ -636,10 +639,10 @@ public class IndexBrowser_p {
// generate outbound-links table
if (outboundHosts.size() > 0) {
maxcount = 200;
- ReversibleScoreMap<String> score = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);
- for (Map.Entry<String, ReversibleScoreMap<String>> entry: outboundHosts.entrySet()) score.set(entry.getKey(), entry.getValue().size());
+ final ReversibleScoreMap<String> score = new ClusteredScoreMap<String>(UTF8.insensitiveUTF8Comparator);
+ for (final Map.Entry<String, ReversibleScoreMap<String>> entry: outboundHosts.entrySet()) score.set(entry.getKey(), entry.getValue().size());
c = 0;
- Iterator<String> i = score.keys(false);
+ final Iterator<String> i = score.keys(false);
while (i.hasNext() && c < maxcount) {
host = i.next();
prop.putHTML("outbound_list_" + c + "_host", host);
@@ -657,10 +660,10 @@ public class IndexBrowser_p {
ConcurrentLog.logException(e);
}
- // for addtoblacklist button
+ // for addtoblacklist button
if (post.containsKey("addtoblacklist") && sb != null) {
final File blacklistsPath = sb.getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT);
- String blacklistname = Blacklist.defaultBlacklist(blacklistsPath);
+ final String blacklistname = Blacklist.defaultBlacklist(blacklistsPath);
if (blacklistname != null) {
String hoststr = pathURI.getHost();
if (hoststr.startsWith("www.")) { // to allow the automatic matching for www.host.com and host.com
@@ -668,7 +671,7 @@ public class IndexBrowser_p {
}
final boolean isok = addBlacklistEntry(
blacklistname,
- hoststr + "/.*"); // as yacy expands host pattern to match www.host.com as well as host.com no need to prefix with "*."
+ hoststr + "/.*"); // as yacy expands host pattern to match www.host.com as well as host.com no need to prefix with "*."
if (isok) {
prop.put("files_blkadded", 1); // glyphicon-ok on (display)
} else {
@@ -690,14 +693,14 @@ public class IndexBrowser_p {
public Double cr_c;
public int crawldepth, references, references_internal, references_external, references_exthosts;
public List<String> references_internal_urls, references_external_urls;
- public InfoCacheEntry(final Fulltext fulltext, final ReferenceReportCache rrCache, final SolrDocument doc, final String urlhash, boolean fetchReferences) {
+ public InfoCacheEntry(final Fulltext fulltext, final ReferenceReportCache rrCache, final SolrDocument doc, final String urlhash, final boolean fetchReferences) {
this.cr_c = (Double) doc.getFieldValue(CollectionSchema.cr_host_chance_d.getSolrFieldName());
this.cr_n = (Integer) doc.getFieldValue(CollectionSchema.cr_host_norm_i.getSolrFieldName());
- Integer cr = (Integer) doc.getFieldValue(CollectionSchema.crawldepth_i.getSolrFieldName());
- Integer rc = (Integer) doc.getFieldValue(CollectionSchema.references_i.getSolrFieldName());
- Integer rc_internal = (Integer) doc.getFieldValue(CollectionSchema.references_internal_i.getSolrFieldName());
- Integer rc_external = (Integer) doc.getFieldValue(CollectionSchema.references_external_i.getSolrFieldName());
- Integer rc_exthosts = (Integer) doc.getFieldValue(CollectionSchema.references_exthosts_i.getSolrFieldName());
+ final Integer cr = (Integer) doc.getFieldValue(CollectionSchema.crawldepth_i.getSolrFieldName());
+ final Integer rc = (Integer) doc.getFieldValue(CollectionSchema.references_i.getSolrFieldName());
+ final Integer rc_internal = (Integer) doc.getFieldValue(CollectionSchema.references_internal_i.getSolrFieldName());
+ final Integer rc_external = (Integer) doc.getFieldValue(CollectionSchema.references_external_i.getSolrFieldName());
+ final Integer rc_exthosts = (Integer) doc.getFieldValue(CollectionSchema.references_exthosts_i.getSolrFieldName());
this.crawldepth = (cr == null || cr.intValue() < 0) ? 0 : cr.intValue(); // for lazy value storage; non-existent means: stored as '0'
this.references = (rc == null || rc.intValue() <= 0) ? 0 : rc.intValue();
this.references_internal = (rc_internal == null || rc_internal.intValue() <= 0) ? 0 : rc_internal.intValue();
@@ -707,22 +710,22 @@ public class IndexBrowser_p {
if (fetchReferences) {
// get the references from the citation index
try {
- ReferenceReport rr = rrCache.getReferenceReport(urlhash, false);
- List<String> internalIDs = new ArrayList<String>();
- List<String> externalIDs = new ArrayList<String>();
- HandleSet iids = rr.getInternallIDs();
- for (byte[] b: iids) internalIDs.add(ASCII.String(b));
- HandleSet eids = rr.getExternalIDs();
- for (byte[] b: eids) externalIDs.add(ASCII.String(b));
+ final ReferenceReport rr = rrCache.getReferenceReport(urlhash, false);
+ final List<String> internalIDs = new ArrayList<String>();
+ final List<String> externalIDs = new ArrayList<String>();
+ final HandleSet iids = rr.getInternallIDs();
+ for (final byte[] b: iids) internalIDs.add(ASCII.String(b));
+ final HandleSet eids = rr.getExternalIDs();
+ for (final byte[] b: eids) externalIDs.add(ASCII.String(b));
// get all urls from the index and store them here
- for (String id: internalIDs) {
+ for (final String id: internalIDs) {
if (id.equals(urlhash)) continue; // no self-references
- String u = fulltext.getURL(id);
+ final String u = fulltext.getURL(id);
if (u != null) this.references_internal_urls.add(u);
}
- for (String id: externalIDs) {
+ for (final String id: externalIDs) {
if (id.equals(urlhash)) continue; // no self-references
- String u = fulltext.getURL(id);
+ final String u = fulltext.getURL(id);
if (u != null) this.references_external_urls.add(u);
}
} catch (final IOException e) {
@@ -733,17 +736,17 @@ public class IndexBrowser_p {
}
@Override
public String toString() {
- StringBuilder sbi = new StringBuilder();
+ final StringBuilder sbi = new StringBuilder();
int c = 0;
- for (String s: this.references_internal_urls) {
+ for (final String s: this.references_internal_urls) {
sbi.append("<a href='").append(s).append("' target='_blank'><img src='env/grafics/i16.gif' alt='info' title='" + s + "' width='12' height='12'/></a>");
c++;
if (c % 80 == 0) sbi.append("<br/>");
}
if (sbi.length() > 0) sbi.insert(0, "<br/>internal referrer:");
- StringBuilder sbe = new StringBuilder();
+ final StringBuilder sbe = new StringBuilder();
c = 0;
- for (String s: this.references_external_urls) {
+ for (final String s: this.references_external_urls) {
sbe.append("<a href='").append(s).append("' target='_blank'><img src='env/grafics/i16.gif' alt='info' title='" + s + "' width='12' height='12'/></a>");
c++;
if (c % 80 == 0) sbe.append("<br/>");
diff --git a/htroot/IndexControlRWIs_p.java b/source/net/yacy/htroot/IndexControlRWIs_p.java
similarity index 98%
rename from htroot/IndexControlRWIs_p.java
rename to source/net/yacy/htroot/IndexControlRWIs_p.java
index a66b9fe27..ec5b03f28 100644
--- a/htroot/IndexControlRWIs_p.java
+++ b/source/net/yacy/htroot/IndexControlRWIs_p.java
@@ -25,6 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -107,10 +109,10 @@ public class IndexControlRWIs_p {
// clean up all search events
SearchEventCache.cleanupEvents(true);
- Segment segment = sb.index;
+ final Segment segment = sb.index;
if ( post != null ) {
- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
final String keystring = post.get("keystring", "").trim();
byte[] keyhash = post.get("keyhash", "").trim().getBytes();
if (keystring.length() > 0 && !keystring.contains(errmsg)) {
@@ -304,7 +306,7 @@ public class IndexControlRWIs_p {
Reference iEntry;
while (urlIter.hasNext()) {
iEntry = urlIter.next();
- boolean exists = segment.fulltext().exists(ASCII.String(iEntry.urlhash()));
+ final boolean exists = segment.fulltext().exists(ASCII.String(iEntry.urlhash()));
if (exists) {
try {
knownURLs.put(iEntry.urlhash());
@@ -404,13 +406,13 @@ public class IndexControlRWIs_p {
ConcurrentLog.logException(e);
}
try {
- String u = segment.fulltext().getURL(ASCII.String(b));
+ final String u = segment.fulltext().getURL(ASCII.String(b));
url = u == null ? null : new DigestURL(u);
segment.fulltext().remove(b);
if ( url != null ) {
items.add(new BlacklistHostAndPath(url.getHost(), url.getFile()));
}
- } catch (IOException e1) {
+ } catch (final IOException e1) {
ConcurrentLog.logException(e1);
}
}
@@ -421,7 +423,7 @@ public class IndexControlRWIs_p {
try {
Switchboard.urlBlacklist.add(supportedBlacklistType,
blacklist, items);
- } catch (PunycodeException e) {
+ } catch (final PunycodeException e) {
ConcurrentLog.warn(APP_NAME,
"Unable to add blacklist entries to blacklist "
+ supportedBlacklistType, e);
@@ -440,7 +442,7 @@ public class IndexControlRWIs_p {
ConcurrentLog.logException(e);
}
try {
- String u = segment.fulltext().getURL(ASCII.String(b));
+ final String u = segment.fulltext().getURL(ASCII.String(b));
url = u == null ? null : new DigestURL(u);
segment.fulltext().remove(b);
if ( url != null ) {
@@ -460,7 +462,7 @@ public class IndexControlRWIs_p {
}
}
}
- } catch (IOException e1) {
+ } catch (final IOException e1) {
ConcurrentLog.logException(e1);
}
}
diff --git a/htroot/IndexControlURLs_p.java b/source/net/yacy/htroot/IndexControlURLs_p.java
similarity index 92%
rename from htroot/IndexControlURLs_p.java
rename to source/net/yacy/htroot/IndexControlURLs_p.java
index 6772c414d..72594e912 100644
--- a/htroot/IndexControlURLs_p.java
+++ b/source/net/yacy/htroot/IndexControlURLs_p.java
@@ -21,6 +21,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
@@ -63,8 +65,8 @@ public class IndexControlURLs_p {
final String nextTransactionToken = TransactionManager.getTransactionToken(header);
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken);
- Segment segment = sb.index;
- long ucount = segment.fulltext().collectionSize();
+ final Segment segment = sb.index;
+ final long ucount = segment.fulltext().collectionSize();
// set default values
prop.put("urlstring", "");
@@ -79,7 +81,7 @@ public class IndexControlURLs_p {
prop.put("reload", 0);
prop.put("dumprestore", 1);
prop.put("dumprestore_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken);
- List<File> dumpFiles = segment.fulltext().dumpFiles();
+ final List<File> dumpFiles = segment.fulltext().dumpFiles();
prop.put("dumprestore_dumpfile", dumpFiles.size() == 0 ? "" : dumpFiles.get(dumpFiles.size() - 1).getAbsolutePath());
prop.put("dumprestore_optimizemax", 10);
prop.put("dumprestore_rebootSolrEnabled",
@@ -163,9 +165,9 @@ public class IndexControlURLs_p {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
- int i = segment.removeAllUrlReferences(urlhash.getBytes(), sb.loader, agent, CacheStrategy.IFEXIST);
- try {segment.loadTimeIndex().remove(urlhash.getBytes());} catch (IOException e) {}
+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
+ final int i = segment.removeAllUrlReferences(urlhash.getBytes(), sb.loader, agent, CacheStrategy.IFEXIST);
+ try {segment.loadTimeIndex().remove(urlhash.getBytes());} catch (final IOException e) {}
prop.put("result", "Deleted URL and " + i + " references from " + i + " word indexes.");
}
@@ -184,7 +186,7 @@ public class IndexControlURLs_p {
prop.putHTML("result", "Removed URL " + url);
}
segment.loadTimeIndex().remove(urlhash.getBytes());
- } catch (IOException e) {
+ } catch (final IOException e) {
prop.putHTML("result", "Error when querying the url hash " + urlhash + ":" + e.getMessage());
}
}
@@ -202,7 +204,7 @@ public class IndexControlURLs_p {
prop.put("result", "No input given; nothing deleted.");
} else {
sb.urlRemove(segment, urlhash.getBytes());
- try {segment.loadTimeIndex().remove(urlhash.getBytes());} catch (IOException e) {}
+ try {segment.loadTimeIndex().remove(urlhash.getBytes());} catch (final IOException e) {}
prop.putHTML("result", "Removed URL " + urlstring);
}
}
@@ -266,10 +268,10 @@ public class IndexControlURLs_p {
TransactionManager.checkPostTransaction(header, post);
final String domain = post.get("domain");
- Set<String> hostnames = new HashSet<String>();
+ final Set<String> hostnames = new HashSet<String>();
hostnames.add(domain);
segment.fulltext().deleteStaleDomainNames(hostnames, null);
- try {segment.loadTimeIndex().clear();} catch (IOException e) {} // delete all to prevent that existing entries reject reloading
+ try {segment.loadTimeIndex().clear();} catch (final IOException e) {} // delete all to prevent that existing entries reject reloading
// trigger the loading of the table
post.put("statistics", "");
}
@@ -280,9 +282,9 @@ public class IndexControlURLs_p {
int cnt = 0;
try {
final Fulltext metadata = segment.fulltext();
- Map<String, ReversibleScoreMap<String>> scores = metadata.getDefaultConnector().getFacets(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", count, CollectionSchema.host_s.getSolrFieldName());
- ReversibleScoreMap<String> stats = scores.get(CollectionSchema.host_s.getSolrFieldName());
- Iterator<String> statsiter = stats.keys(false);
+ final Map<String, ReversibleScoreMap<String>> scores = metadata.getDefaultConnector().getFacets(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", count, CollectionSchema.host_s.getSolrFieldName());
+ final ReversibleScoreMap<String> stats = scores.get(CollectionSchema.host_s.getSolrFieldName());
+ final Iterator<String> statsiter = stats.keys(false);
boolean dark = true;
String hostname;
prop.put("statisticslines_domains_" + cnt + "lines", count);
diff --git a/htroot/IndexCreateLoaderQueue_p.java b/source/net/yacy/htroot/IndexCreateLoaderQueue_p.java
similarity index 92%
rename from htroot/IndexCreateLoaderQueue_p.java
rename to source/net/yacy/htroot/IndexCreateLoaderQueue_p.java
index 331e46cca..377107cab 100644
--- a/htroot/IndexCreateLoaderQueue_p.java
+++ b/source/net/yacy/htroot/IndexCreateLoaderQueue_p.java
@@ -27,6 +27,8 @@
// javac -classpath .:../classes IndexCreate_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.Map;
import net.yacy.cora.document.encoding.ASCII;
@@ -45,7 +47,7 @@ public class IndexCreateLoaderQueue_p {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
- Map<DigestURL, Request> map = sb.crawlQueues.activeWorkerEntries();
+ final Map<DigestURL, Request> map = sb.crawlQueues.activeWorkerEntries();
if (map.size() == 0) {
prop.put("loader-set", "0");
} else {
@@ -53,7 +55,7 @@ public class IndexCreateLoaderQueue_p {
boolean dark = true;
Seed initiator;
int count = 0;
- for (Request element : map.values()) {
+ for (final Request element : map.values()) {
if (element == null) continue;
initiator = sb.peers.getConnected((element.initiator() == null) ? "" : ASCII.String(element.initiator()));
diff --git a/htroot/IndexCreateParserErrors_p.java b/source/net/yacy/htroot/IndexCreateParserErrors_p.java
similarity index 93%
rename from htroot/IndexCreateParserErrors_p.java
rename to source/net/yacy/htroot/IndexCreateParserErrors_p.java
index 11dc9d075..0840be2e0 100644
--- a/htroot/IndexCreateParserErrors_p.java
+++ b/source/net/yacy/htroot/IndexCreateParserErrors_p.java
@@ -23,6 +23,8 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.ArrayList;
import net.yacy.cora.date.GenericFormatter;
@@ -72,18 +74,18 @@ public class IndexCreateParserErrors_p {
dark = true;
DigestURL url;
int j=0;
- ArrayList<CollectionConfiguration.FailDoc> l = sb.crawlQueues.errorURL.list(showRejectedCount);
+ final ArrayList<CollectionConfiguration.FailDoc> l = sb.crawlQueues.errorURL.list(showRejectedCount);
CollectionConfiguration.FailDoc entry;
for (int i = l.size() - 1; i >= 0; i--) {
entry = l.get(i);
if (entry == null) continue;
url = entry.getDigestURL();
if (url == null) continue;
-
+
prop.putHTML("rejected_list_"+j+"_time", GenericFormatter.SIMPLE_FORMATTER.format(entry.getFailDate()));
prop.putHTML("rejected_list_"+j+"_url", url.toNormalform(false));
-
- String cause = entry.getFailReason();
+
+ final String cause = entry.getFailReason();
if (cause == null) {
// well that should not really happen but occurs in a specific combination of running crawls for same domains with different url filters
prop.put("rejected_list_"+j+"_failreason", "no fail reason given");
diff --git a/htroot/IndexCreateQueues_p.java b/source/net/yacy/htroot/IndexCreateQueues_p.java
similarity index 91%
rename from htroot/IndexCreateQueues_p.java
rename to source/net/yacy/htroot/IndexCreateQueues_p.java
index 766a9e635..f4ae0e7e9 100644
--- a/htroot/IndexCreateQueues_p.java
+++ b/source/net/yacy/htroot/IndexCreateQueues_p.java
@@ -1,22 +1,24 @@
+package net.yacy.htroot;
+
import java.net.MalformedURLException;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Collections;
-import java.util.stream.Collectors;
-import java.util.LinkedHashMap;
-import java.util.TreeMap;
import java.util.Set;
+import java.util.TreeMap;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
import org.apache.http.conn.util.InetAddressUtils;
@@ -39,7 +41,7 @@ public class IndexCreateQueues_p {
private static final DateTimeFormatter DAY_FORMATTER = DateTimeFormatter
.ofPattern("uuuu/MM/dd", Locale.US).withZone(ZoneId.systemDefault());
-
+
/**
* @param date a date to render as a String
* @return the date formatted using the DAY_FORMATTER pattern.
@@ -76,7 +78,7 @@ public class IndexCreateQueues_p {
if (post.containsKey("embed")) embed = true;
if (post.containsKey("sort")) {
- String countSort = post.get("sort");
+ final String countSort = post.get("sort");
if (countSort.equals("count")) sortByCount = +1;
if (countSort.equals("-count")) sortByCount = -1;
if (countSort.equals("host")) sortByHost = +1;
@@ -110,16 +112,16 @@ public class IndexCreateQueues_p {
int removedByHosts = 0;
if (option == URL && deletepattern.startsWith(".*") && deletepattern.endsWith(".*")) {
// try to delete that using the host name
- Set<String> hosthashes = new HashSet<String>();
- String hn = deletepattern.substring(2, deletepattern.length() - 2);
+ final Set<String> hosthashes = new HashSet<String>();
+ final String hn = deletepattern.substring(2, deletepattern.length() - 2);
try {
hosthashes.add(DigestURL.hosthash(hn, hn.startsWith("ftp") ? 21 : 80));
hosthashes.add(DigestURL.hosthash(hn, 443));
removedByHosts = sb.crawlQueues.removeHosts(hosthashes);
- } catch (MalformedURLException e) {
+ } catch (final MalformedURLException e) {
}
}
-
+
if (removedByHosts == 0) {
// iterating through the list of URLs
final Iterator<Request> iter = sb.crawlQueues.noticeURL.iterator(stackType);
@@ -128,7 +130,7 @@ public class IndexCreateQueues_p {
while (iter.hasNext()) {
if ((entry = iter.next()) == null) continue;
String value = null;
-
+
location: switch (option) {
case URL: value = (entry.url() == null) ? null : entry.url().toString(); break location;
case ANCHOR: value = entry.name(); break location;
@@ -139,7 +141,7 @@ public class IndexCreateQueues_p {
case MODIFIED: value = daydate(entry.appdate()); break location;
default: value = null; break location;
}
-
+
if (value != null && compiledPattern.matcher(value).matches()) removehashes.add(entry.url().hash());
}
ConcurrentLog.info("IndexCreateQueues_p", "created a remove list with " + removehashes.size() + " entries for pattern '" + deletepattern + "'");
@@ -155,7 +157,7 @@ public class IndexCreateQueues_p {
}
}
- int stackSize = sb.crawlQueues.noticeURL.stackSize(stackType);
+ final int stackSize = sb.crawlQueues.noticeURL.stackSize(stackType);
if (stackSize == 0) {
prop.put("crawler", "0");
} else {
@@ -182,41 +184,41 @@ public class IndexCreateQueues_p {
prop.put("crawler_showtable_sortedByCount",true);
if (sortByCount < 0) {
prop.put("crawler_showtable_sortedByCount_asc", false);
- Map<String, Integer[]> hosts_sorted = hosts.entrySet().stream()
+ final Map<String, Integer[]> hosts_sorted = hosts.entrySet().stream()
.sorted( (e1, e2)->e2.getValue()[0].compareTo(e1.getValue()[0]) )
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new));
hosts = hosts_sorted;
}
else {
prop.put("crawler_showtable_sortedByCount_asc", true);
- Map<String, Integer[]> hosts_sorted = hosts.entrySet().stream()
+ final Map<String, Integer[]> hosts_sorted = hosts.entrySet().stream()
.sorted( (e1, e2)->e1.getValue()[0].compareTo(e2.getValue()[0]) )
.collect(Collectors.toMap(Entry::getKey, Entry::getValue, (e1, e2) -> e1, LinkedHashMap::new));
hosts = hosts_sorted;
}
}
-
+
if ( sortByHost!=0 ) {
prop.put("crawler_showtable_sortedByCount",false);
prop.put("crawler_showtable_sortedByCount_asc",true);
prop.put("crawler_showtable_sortedByHost",true);
if (sortByHost < 0) {
prop.put("crawler_showtable_sortedByHost_asc", false);
- Map<String, Integer[]> hosts_sorted = new TreeMap<String, Integer[]>(Collections.reverseOrder());
+ final Map<String, Integer[]> hosts_sorted = new TreeMap<String, Integer[]>(Collections.reverseOrder());
hosts_sorted.putAll(hosts);
hosts = hosts_sorted;
}
else {
prop.put("crawler_showtable_sortedByHost_asc", true);
- Map<String, Integer[]> hosts_sorted = new TreeMap<String, Integer[]>(hosts);
+ final Map<String, Integer[]> hosts_sorted = new TreeMap<String, Integer[]>(hosts);
hosts = hosts_sorted;
}
}
}
int hc = 0;
- for (Map.Entry<String, Integer[]> host: hosts.entrySet()) {
- String hostnameport = host.getKey();
+ for (final Map.Entry<String, Integer[]> host: hosts.entrySet()) {
+ final String hostnameport = host.getKey();
String hostname = Domains.stripToHostName(hostnameport);
if(InetAddressUtils.isIPv6Address(hostname)) {
hostname = "[" + hostname + "]"; // HostBalancer.getDomainStackReferences() function requires square brackets around IPV6 addresses
@@ -228,13 +230,13 @@ public class IndexCreateQueues_p {
prop.putHTML("crawler_host_" + hc + "_queuename", stackType.name());
prop.put("crawler_host_" + hc + "_hostcount", host.getValue()[0]);
prop.put("crawler_host_" + hc + "_hostdelta", host.getValue()[1] == Integer.MIN_VALUE ? "not accessed" : Integer.toString(host.getValue()[1]));
- List<Request> domainStackReferences = sb.crawlQueues.noticeURL.getDomainStackReferences(stackType, hostname, urlsPerHost, 10000);
+ final List<Request> domainStackReferences = sb.crawlQueues.noticeURL.getDomainStackReferences(stackType, hostname, urlsPerHost, 10000);
Seed initiator;
String profileHandle;
CrawlProfile profileEntry;
int count = 0;
- for (Request request: domainStackReferences) {
+ for (final Request request: domainStackReferences) {
if (request == null) continue;
initiator = sb.peers.getConnected(request.initiator() == null ? "" : ASCII.String(request.initiator()));
profileHandle = request.profileHandle();
diff --git a/htroot/IndexDeletion_p.java b/source/net/yacy/htroot/IndexDeletion_p.java
similarity index 82%
rename from htroot/IndexDeletion_p.java
rename to source/net/yacy/htroot/IndexDeletion_p.java
index 3b437a7d8..99aed1077 100644
--- a/htroot/IndexDeletion_p.java
+++ b/source/net/yacy/htroot/IndexDeletion_p.java
@@ -18,6 +18,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Date;
@@ -56,8 +58,8 @@ public class IndexDeletion_p {
/* Acquire a transaction token for the next POST form submission */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));
- SolrConnector defaultConnector = sb.index.fulltext().getDefaultConnector();
- SolrConnector webgraphConnector = sb.index.fulltext().getWebgraphConnector();
+ final SolrConnector defaultConnector = sb.index.fulltext().getDefaultConnector();
+ final SolrConnector webgraphConnector = sb.index.fulltext().getWebgraphConnector();
if (post == null || post.size() == 0) defaultConnector.commit(false); // we must do a commit here because the user cannot see a proper count.
String schemaName = CollectionSchema.CORE_NAME;
@@ -65,16 +67,16 @@ public class IndexDeletion_p {
// Delete by URL Matching
String urldelete = post == null ? "" : post.get("urldelete", "");
- boolean urldelete_mm_subpath_checked = post == null ? true : post.get("urldelete-mm", "subpath").equals("subpath");
+ final boolean urldelete_mm_subpath_checked = post == null ? true : post.get("urldelete-mm", "subpath").equals("subpath");
prop.putHTML("urldelete", urldelete);
prop.put("urldelete-mm-subpath-checked", urldelete_mm_subpath_checked ? 1 : 0);
prop.put("urldelete-mm-regexp-checked", urldelete_mm_subpath_checked ? 0 : 1);
prop.put("urldelete-active", 0);
// Delete by Age
- int timedelete_number = post == null ? 14 : post.getInt("timedelete-number", 14);
- String timedelete_unit = post == null ? "day" : post.get("timedelete-unit", "day");
- boolean timedelete_source_loaddate_checked = post == null ? true : post.get("timedelete-source", "loaddate").equals("loaddate");
+ final int timedelete_number = post == null ? 14 : post.getInt("timedelete-number", 14);
+ final String timedelete_unit = post == null ? "day" : post.get("timedelete-unit", "day");
+ final boolean timedelete_source_loaddate_checked = post == null ? true : post.get("timedelete-source", "loaddate").equals("loaddate");
for (int i = 1; i <= 90; i++) prop.put("timedelete-n-" + i, 0);
prop.put("timedelete-n-" + timedelete_number, timedelete_number);
prop.put("timedelete-u-year", timedelete_unit.equals("year") ? 1 : 0);
@@ -92,11 +94,11 @@ public class IndexDeletion_p {
collectiondelete_mode_unassigned_checked = false;
prop.put("collectiondelete-select", 1);
try {
- ScoreMap<String> collectionMap = defaultConnector.getFacets("*:*", 1000, CollectionSchema.collection_sxt.getSolrFieldName()).get(CollectionSchema.collection_sxt.getSolrFieldName());
- Iterator<String> i = collectionMap.iterator();
+ final ScoreMap<String> collectionMap = defaultConnector.getFacets("*:*", 1000, CollectionSchema.collection_sxt.getSolrFieldName()).get(CollectionSchema.collection_sxt.getSolrFieldName());
+ final Iterator<String> i = collectionMap.iterator();
int c = 0;
while (i.hasNext()) {
- String collection = i.next();
+ final String collection = i.next();
prop.put("collectiondelete-select_list_" + c + "_collection-name", collection + "/" + collectionMap.get(collection));
prop.put("collectiondelete-select_list_" + c + "_collection-value", collection);
c++;
@@ -128,26 +130,26 @@ public class IndexDeletion_p {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
- boolean simulate = post.containsKey("simulate-urldelete");
+ final boolean simulate = post.containsKey("simulate-urldelete");
// parse the input
urldelete = urldelete.trim();
if (urldelete_mm_subpath_checked) {
// collect using url stubs
- Set<String> ids = new HashSet<String>();
- String[] stubURLs = urldelete.indexOf('\n') > 0 || urldelete.indexOf('\r') > 0 ? urldelete.split("[\\r\\n]+") : urldelete.split(Pattern.quote("|"));
+ final Set<String> ids = new HashSet<String>();
+ final String[] stubURLs = urldelete.indexOf('\n') > 0 || urldelete.indexOf('\r') > 0 ? urldelete.split("[\\r\\n]+") : urldelete.split(Pattern.quote("|"));
for (String urlStub: stubURLs) {
if (urlStub == null || urlStub.length() == 0) continue;
- int pos = urlStub.indexOf("://",0);
+ final int pos = urlStub.indexOf("://",0);
if (pos == -1) {
if (urlStub.startsWith("ftp")) urlStub = "ftp://" + urlStub; else urlStub = "http://" + urlStub;
}
try {
- DigestURL u = new DigestURL(urlStub);
- BlockingQueue<SolrDocument> dq = defaultConnector.concurrentDocumentsByQuery(CollectionSchema.host_s.getSolrFieldName() + ":\"" + u.getHost() + "\"", null, 0, 100000000, Long.MAX_VALUE, 100, 1, false, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
+ final DigestURL u = new DigestURL(urlStub);
+ final BlockingQueue<SolrDocument> dq = defaultConnector.concurrentDocumentsByQuery(CollectionSchema.host_s.getSolrFieldName() + ":\"" + u.getHost() + "\"", null, 0, 100000000, Long.MAX_VALUE, 100, 1, false, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName());
SolrDocument doc;
try {
while ((doc = dq.take()) != AbstractSolrConnector.POISON_DOCUMENT) {
- String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
+ final String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
if (url.startsWith(urlStub)) ids.add((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()));
}
} catch (final InterruptedException e) {
@@ -161,13 +163,13 @@ public class IndexDeletion_p {
} else {
sb.remove(ids);
defaultConnector.commit(false);
- ids.forEach(urlhash -> {try {sb.index.loadTimeIndex().remove(urlhash.getBytes());} catch (IOException e) {}});
+ ids.forEach(urlhash -> {try {sb.index.loadTimeIndex().remove(urlhash.getBytes());} catch (final IOException e) {}});
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs matching with " + urldelete);
prop.put("urldelete-active", 2);
}
} else {
// collect using a regular expression on urls
- String regexquery = CollectionSchema.sku.getSolrFieldName() + ":/" + urldelete + "/";
+ final String regexquery = CollectionSchema.sku.getSolrFieldName() + ":/" + urldelete + "/";
if (simulate) {
try {
count = (int) defaultConnector.getCountByQuery("{!cache=false}" + regexquery);
@@ -178,7 +180,7 @@ public class IndexDeletion_p {
try {
defaultConnector.deleteByQuery(regexquery);
defaultConnector.commit(false);
- try {sb.index.loadTimeIndex().clear();} catch (IOException e) {}
+ try {sb.index.loadTimeIndex().clear();} catch (final IOException e) {}
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, regex match = " + urldelete);
} catch (final IOException e) {
}
@@ -192,9 +194,9 @@ public class IndexDeletion_p {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
- boolean simulate = post.containsKey("simulate-timedelete");
+ final boolean simulate = post.containsKey("simulate-timedelete");
Date deleteageDate = null;
- long t = timeParser(timedelete_number, timedelete_unit); // year, month, day, hour
+ final long t = timeParser(timedelete_number, timedelete_unit); // year, month, day, hour
if (t > 0) deleteageDate = new Date(t);
final String collection1Query = (timedelete_source_loaddate_checked ? CollectionSchema.load_date_dt : CollectionSchema.last_modified).getSolrFieldName() + ":[* TO " + ISO8601Formatter.FORMATTER.format(deleteageDate) + "]";
final String webgraphQuery = (timedelete_source_loaddate_checked ? WebgraphSchema.load_date_dt : WebgraphSchema.last_modified).getSolrFieldName() + ":[* TO " + ISO8601Formatter.FORMATTER.format(deleteageDate) + "]";
@@ -208,7 +210,7 @@ public class IndexDeletion_p {
try {
defaultConnector.deleteByQuery(collection1Query);
defaultConnector.commit(false);
- try {sb.index.loadTimeIndex().clear();} catch (IOException e) {}
+ try {sb.index.loadTimeIndex().clear();} catch (final IOException e) {}
if (webgraphConnector != null) webgraphConnector.deleteByQuery(webgraphQuery);
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, docs older than " + timedelete_number + " " + timedelete_unit);
} catch (final IOException e) {
@@ -222,9 +224,9 @@ public class IndexDeletion_p {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
- boolean simulate = post.containsKey("simulate-collectiondelete");
+ final boolean simulate = post.containsKey("simulate-collectiondelete");
collectiondelete = collectiondelete.replaceAll(" ","").replaceAll(",", "|");
- String query = collectiondelete_mode_unassigned_checked ? "-" + CollectionSchema.collection_sxt + AbstractSolrConnector.CATCHALL_DTERM : collectiondelete.length() == 0 ? CollectionSchema.collection_sxt + ":\"\"" : QueryModifier.parseCollectionExpression(collectiondelete);
+ final String query = collectiondelete_mode_unassigned_checked ? "-" + CollectionSchema.collection_sxt + AbstractSolrConnector.CATCHALL_DTERM : collectiondelete.length() == 0 ? CollectionSchema.collection_sxt + ":\"\"" : QueryModifier.parseCollectionExpression(collectiondelete);
if (simulate) {
try {
count = (int) defaultConnector.getCountByQuery(query);
@@ -235,7 +237,7 @@ public class IndexDeletion_p {
try {
defaultConnector.deleteByQuery(query);
defaultConnector.commit(false);
- try {sb.index.loadTimeIndex().clear();} catch (IOException e) {}
+ try {sb.index.loadTimeIndex().clear();} catch (final IOException e) {}
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, collection " + collectiondelete);
} catch (final IOException e) {
}
@@ -248,9 +250,9 @@ public class IndexDeletion_p {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
- boolean simulate = post.containsKey("simulate-querydelete");
+ final boolean simulate = post.containsKey("simulate-querydelete");
- SolrConnector connector = schemaName.equals(CollectionSchema.CORE_NAME) ? defaultConnector : sb.index.fulltext().getWebgraphConnector();
+ final SolrConnector connector = schemaName.equals(CollectionSchema.CORE_NAME) ? defaultConnector : sb.index.fulltext().getWebgraphConnector();
if (simulate) {
try {
count = (int) connector.getCountByQuery(querydelete);
@@ -262,7 +264,7 @@ public class IndexDeletion_p {
ConcurrentLog.info("IndexDeletion", "delete by query \"" + querydelete + "\", size before deletion = " + connector.getSize());
connector.deleteByQuery(querydelete);
connector.commit(false);
- try {sb.index.loadTimeIndex().clear();} catch (IOException e) {}
+ try {sb.index.loadTimeIndex().clear();} catch (final IOException e) {}
ConcurrentLog.info("IndexDeletion", "delete by query \"" + querydelete + "\", size after commit = " + connector.getSize());
sb.tables.recordAPICall(post, "IndexDeletion_p.html", WorkTables.TABLE_API_TYPE_DELETION, "deletion, solr query, q = " + querydelete);
} catch (final IOException e) {
diff --git a/htroot/IndexExport_p.java b/source/net/yacy/htroot/IndexExport_p.java
similarity index 97%
rename from htroot/IndexExport_p.java
rename to source/net/yacy/htroot/IndexExport_p.java
index 2ef08ea6c..78cc94132 100644
--- a/htroot/IndexExport_p.java
+++ b/source/net/yacy/htroot/IndexExport_p.java
@@ -21,6 +21,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.util.List;
@@ -46,13 +48,13 @@ public class IndexExport_p {
final serverObjects prop = new serverObjects();
- Segment segment = sb.index;
+ final Segment segment = sb.index;
// we have two counts of document: total number and such that are exportable with status code 200
- long ucount = segment.fulltext().collectionSize();
+ final long ucount = segment.fulltext().collectionSize();
long ucount200 = ucount;
try {
ucount200 = segment.fulltext().getDefaultConnector().getCountByQuery(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200");
- } catch (IOException e1) {}
+ } catch (final IOException e1) {}
// set default values
prop.put("otherHosts", "");
@@ -129,7 +131,7 @@ public class IndexExport_p {
// start the export
try {
export = sb.index.fulltext().export(format, filter, query, maxseconds, new File(path), dom, text);
- } catch (IOException e) {
+ } catch (final IOException e) {
prop.put("lurlexporterror", 1);
prop.put("lurlexporterror_exportfile", "-no export-");
prop.put("lurlexporterror_exportfailmsg", e.getMessage());
diff --git a/htroot/IndexFederated_p.java b/source/net/yacy/htroot/IndexFederated_p.java
similarity index 90%
rename from htroot/IndexFederated_p.java
rename to source/net/yacy/htroot/IndexFederated_p.java
index 557d5d33b..fbe9de77b 100644
--- a/htroot/IndexFederated_p.java
+++ b/source/net/yacy/htroot/IndexFederated_p.java
@@ -18,6 +18,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -47,13 +49,13 @@ public class IndexFederated_p {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
-
+
if (post != null && post.containsKey("setrwi")) {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
-
+
//yacy
- boolean post_core_rwi = post.getBoolean("core.service.rwi");
+ final boolean post_core_rwi = post.getBoolean("core.service.rwi");
final boolean previous_core_rwi = sb.index.connectedRWI() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false);
env.setConfig(SwitchboardConstants.CORE_SERVICE_RWI, post_core_rwi);
if (previous_core_rwi && !post_core_rwi) sb.index.disconnectRWI(); // switch off
@@ -67,8 +69,8 @@ public class IndexFederated_p {
if (post != null && post.containsKey("setcitation")) {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
-
- boolean post_core_citation = post.getBoolean(SwitchboardConstants.CORE_SERVICE_CITATION);
+
+ final boolean post_core_citation = post.getBoolean(SwitchboardConstants.CORE_SERVICE_CITATION);
final boolean previous_core_citation = sb.index.connectedCitation() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false);
env.setConfig(SwitchboardConstants.CORE_SERVICE_CITATION, post_core_citation);
if (previous_core_citation && !post_core_citation) sb.index.disconnectCitation(); // switch off
@@ -77,16 +79,16 @@ public class IndexFederated_p {
final long fileSizeMax = (OS.isWindows) ? sb.getConfigLong("filesize.max.win", Integer.MAX_VALUE) : sb.getConfigLong( "filesize.max.other", Integer.MAX_VALUE);
sb.index.connectCitation(wordCacheMaxCount, fileSizeMax);
} catch (final IOException e) { ConcurrentLog.logException(e); } // switch on
- boolean webgraph = post.getBoolean(SwitchboardConstants.CORE_SERVICE_WEBGRAPH);
+ final boolean webgraph = post.getBoolean(SwitchboardConstants.CORE_SERVICE_WEBGRAPH);
sb.index.fulltext().setUseWebgraph(webgraph);
env.setConfig(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, webgraph);
}
-
+
if (post != null && post.containsKey("setsolr")) {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
-
- boolean post_core_fulltext = post.getBoolean(SwitchboardConstants.CORE_SERVICE_FULLTEXT);
+
+ final boolean post_core_fulltext = post.getBoolean(SwitchboardConstants.CORE_SERVICE_FULLTEXT);
final boolean previous_core_fulltext = sb.index.fulltext().connectedLocalSolr() && env.getConfigBool(
SwitchboardConstants.CORE_SERVICE_FULLTEXT, SwitchboardConstants.CORE_SERVICE_FULLTEXT_DEFAULT);
env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext);
@@ -99,7 +101,7 @@ public class IndexFederated_p {
// switch on
try { sb.index.fulltext().connectLocalSolr(); } catch (final IOException e) { ConcurrentLog.logException(e); }
}
-
+
// solr
final boolean solrRemoteWasOn = sb.index.fulltext().connectedRemoteSolr()
&& env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED,
@@ -107,10 +109,10 @@ public class IndexFederated_p {
String solrurls = post.get("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr"));
final boolean solrRemoteIsOnAfterwards = post.getBoolean("solr.indexing.solrremote") & solrurls.length() > 0;
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, solrRemoteIsOnAfterwards);
-
+
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED, post
.getBoolean(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED));
-
+
final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(solrurls))));
final StringBuilder s = new StringBuilder();
String s0;
@@ -128,10 +130,10 @@ public class IndexFederated_p {
}
solrurls = s.toString().trim();
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, solrurls);
- String shardMethodName = post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name()));
- ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
+ final String shardMethodName = post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name()));
+ final ShardSelection.Method shardMethod = ShardSelection.Method.valueOf(shardMethodName);
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, shardMethod.name());
-
+
if (solrRemoteWasOn && !solrRemoteIsOnAfterwards) {
// switch off
try {
@@ -140,7 +142,7 @@ public class IndexFederated_p {
ConcurrentLog.logException(e);
}
}
-
+
final boolean writeEnabled = post.getBoolean("solr.indexing.solrremote.writeenabled");
sb.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, writeEnabled);
@@ -155,10 +157,10 @@ public class IndexFederated_p {
final boolean trustSelfSignedOnAuthenticatedServer = Switchboard.getSwitchboard().getConfigBool(
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT);
-
+
try {
if (usesolr) {
- ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
+ final ArrayList<RemoteInstance> instances = RemoteInstance.getShardInstances(solrurls, null, null, solrtimeout, trustSelfSignedOnAuthenticatedServer);
sb.index.fulltext().connectRemoteSolr(instances, shardMethod, writeEnabled);
} else {
sb.index.fulltext().disconnectRemoteSolr();
@@ -174,11 +176,11 @@ public class IndexFederated_p {
} catch (final SolrException e) {
ConcurrentLog.severe("IndexFederated_p", "change of solr connection failed", e);
}
-
- boolean lazy = post.getBoolean("solr.indexing.lazy");
+
+ final boolean lazy = post.getBoolean("solr.indexing.lazy");
env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy);
}
-
+
/* Acquire a transaction token for the next POST form submission */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));
@@ -189,16 +191,16 @@ public class IndexFederated_p {
prop.put("table", 1);
final SolrConnector solr = sb.index.fulltext().getDefaultRemoteSolrConnector();
final long[] size = new long[]{((RemoteSolrConnector) solr).getSize()};
-
+
final boolean toExternalAddress = !header.accessFromLocalhost();
String externalHost = null;
if(toExternalAddress) {
- /* The request does not come from the same computer than this peer : we get the external address used to reach it from the request headers, and we will use
+ /* The request does not come from the same computer than this peer : we get the external address used to reach it from the request headers, and we will use
* it if some remote solr instance(s) URLs are configured with loopback address alias (such as 'localhost', '127.0.0.1', '[::1]')
* to render the externally reachable Solr instance(s) admin URLs */
externalHost = header.getServerName();
}
-
+
final ArrayList<String> urls = ((ShardInstance) ((RemoteSolrConnector) solr).getInstance())
.getAdminInterfaces(toExternalAddress, externalHost);
boolean dark = false;
@@ -222,10 +224,10 @@ public class IndexFederated_p {
prop.put(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED + ".checked",
env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED,
SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_AUTHENTICATED_ALLOW_SELF_SIGNED_DEFAULT));
- prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
- String thisShardingMethodName = env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
+ prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
+ final String thisShardingMethodName = env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, ShardSelection.Method.MODULO_HOST_MD5.name());
int mc = 0;
- for (ShardSelection.Method method: ShardSelection.Method.values()) {
+ for (final ShardSelection.Method method: ShardSelection.Method.values()) {
prop.put("solr.indexing.sharding.methods_" + mc + "_method", method.name());
prop.put("solr.indexing.sharding.methods_" + mc + "_description", method.description);
prop.put("solr.indexing.sharding.methods_" + mc + "_selected", method.name().equals(thisShardingMethodName) ? 1 : 0);
@@ -234,7 +236,7 @@ public class IndexFederated_p {
prop.put("solr.indexing.sharding.methods", mc);
prop.put("solr.indexing.solrremote.writeenabled.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_WRITEENABLED, true));
prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0);
-
+
// return rewrite properties
return prop;
}
diff --git a/htroot/IndexImportMediawiki_p.java b/source/net/yacy/htroot/IndexImportMediawiki_p.java
similarity index 93%
rename from htroot/IndexImportMediawiki_p.java
rename to source/net/yacy/htroot/IndexImportMediawiki_p.java
index b45a76a5e..faf30ff00 100644
--- a/htroot/IndexImportMediawiki_p.java
+++ b/source/net/yacy/htroot/IndexImportMediawiki_p.java
@@ -22,6 +22,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
@@ -96,18 +98,18 @@ public class IndexImportMediawiki_p {
prop.put("import_prevReport", MediawikiImporter.job != null ? 1 : 0);
if (post == null) {
prop.put("import_status", 0);
-
+
/* Acquire a transaction token for the next POST form submission */
final String token = TransactionManager.getTransactionToken(header);
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, token);
prop.put("import_" + TransactionManager.TRANSACTION_TOKEN_PARAM, token);
-
+
} else {
if (post.containsKey("file")) {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
-
- String file = post.get("file");
+
+ final String file = post.get("file");
MultiProtocolURL sourceURL = null;
int status = 0;
String sourceFilePath = "";
@@ -129,18 +131,18 @@ public class IndexImportMediawiki_p {
status = 4;
}
}
-
+
if (status == 0 && post.getBoolean("iffresh")) {
- long lastModified = getLastModified(sourceURL);
+ final long lastModified = getLastModified(sourceURL);
if (lastExecutionDate != null && lastModified != 0L && Instant.ofEpochMilli(lastModified)
.isBefore(lastExecutionDate.toInstant())) {
status = 5;
prop.put("import_status_lastImportDate", GenericFormatter
.formatSafely(lastExecutionDate.toInstant(), GenericFormatter.FORMAT_SIMPLE));
-
+
/* the import is not performed, but we increase here the api call count */
if(sb.tables != null) {
- byte[] lastExecutedCallPk = lastExecutedCall.getPK();
+ final byte[] lastExecutedCallPk = lastExecutedCall.getPK();
if(lastExecutedCallPk != null && !post.containsKey(WorkTables.TABLE_API_COL_APICALL_PK)) {
post.add(WorkTables.TABLE_API_COL_APICALL_PK, UTF8.String(lastExecutedCallPk));
}
@@ -148,7 +150,7 @@ public class IndexImportMediawiki_p {
}
}
}
- } catch (MalformedURLException e) {
+ } catch (final MalformedURLException e) {
status = 1;
}
if (status == 0) {
@@ -156,14 +158,14 @@ public class IndexImportMediawiki_p {
if(sb.tables != null) {
/* We avoid creating a duplicate of any already recorded API call with the same parameters */
if(lastExecutedCall != null && !post.containsKey(WorkTables.TABLE_API_COL_APICALL_PK)) {
- byte[] lastExecutedCallPk = lastExecutedCall.getPK();
+ final byte[] lastExecutedCallPk = lastExecutedCall.getPK();
if(lastExecutedCallPk != null) {
post.add(WorkTables.TABLE_API_COL_APICALL_PK, UTF8.String(lastExecutedCallPk));
}
}
sb.tables.recordAPICall(post, "IndexImportMediawiki_p.html", WorkTables.TABLE_API_TYPE_DUMP, "MediaWiki Dump Import for " + sourceURL);
}
-
+
MediawikiImporter.job = new MediawikiImporter(sourceURL, sb.surrogatesInPath);
MediawikiImporter.job.start();
prop.put("import_dump", MediawikiImporter.job.source());
@@ -173,7 +175,7 @@ public class IndexImportMediawiki_p {
} else {
prop.put("import_status", status);
prop.put("import_status_sourceFile", sourceFilePath);
-
+
/* Acquire a transaction token for the next POST form submission */
final String token = TransactionManager.getTransactionToken(header);
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, token);
@@ -190,23 +192,23 @@ public class IndexImportMediawiki_p {
}
return prop;
}
-
+
/**
* @param fileURL the file URL. Must not be null.
* @return the last modified date for the file at fileURL, or 0L when unknown or when an error occurred
*/
- private static long getLastModified(MultiProtocolURL fileURL) {
+ private static long getLastModified(final MultiProtocolURL fileURL) {
try {
if (fileURL.isHTTP() || fileURL.isHTTPS()) {
/* http(s) : we do not use MultiprotocolURL.lastModified() which always returns 0L for these protocols */
try (HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent)) {
- HttpResponse headResponse = httpClient.HEADResponse(fileURL, false);
+ final HttpResponse headResponse = httpClient.HEADResponse(fileURL, false);
if (headResponse != null && headResponse.getStatusLine() != null
&& headResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- Header lastModifiedHeader = headResponse
+ final Header lastModifiedHeader = headResponse
.getFirstHeader(HeaderFramework.LAST_MODIFIED);
if (lastModifiedHeader != null) {
- Date lastModifiedDate = HeaderFramework.parseHTTPDate(lastModifiedHeader.getValue());
+ final Date lastModifiedDate = HeaderFramework.parseHTTPDate(lastModifiedHeader.getValue());
if(lastModifiedDate != null) {
return lastModifiedDate.getTime();
}
@@ -216,7 +218,7 @@ public class IndexImportMediawiki_p {
} else {
return fileURL.lastModified();
}
- } catch (IOException ignored) {
+ } catch (final IOException ignored) {
ConcurrentLog.warn("IndexImportMediawiki_p", "Could not retrieve last modified date for dump file at " + fileURL);
}
return 0l;
diff --git a/htroot/IndexImportOAIPMHList_p.java b/source/net/yacy/htroot/IndexImportOAIPMHList_p.java
similarity index 85%
rename from htroot/IndexImportOAIPMHList_p.java
rename to source/net/yacy/htroot/IndexImportOAIPMHList_p.java
index 6c8859540..1aa31bebc 100644
--- a/htroot/IndexImportOAIPMHList_p.java
+++ b/source/net/yacy/htroot/IndexImportOAIPMHList_p.java
@@ -22,6 +22,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -37,7 +39,7 @@ import net.yacy.server.serverSwitch;
public class IndexImportOAIPMHList_p {
- public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, serverObjects post, final serverSwitch env) {
+ public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
@@ -46,7 +48,7 @@ public class IndexImportOAIPMHList_p {
prop.put("source", 0);
if (post != null && post.containsKey("source")) {
- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
final Set<String> oaiRoots = new OAIListFriendsLoader().getListFriends(sb.loader, agent).keySet();
boolean dark = false;
@@ -66,9 +68,9 @@ public class IndexImportOAIPMHList_p {
if (post != null && post.containsKey("import")) {
final List<OAIPMHImporter> jobs = new ArrayList<OAIPMHImporter>();
- for (OAIPMHImporter job: OAIPMHImporter.runningJobs.keySet()) jobs.add(job);
- for (OAIPMHImporter job: OAIPMHImporter.startedJobs.keySet()) jobs.add(job);
- for (OAIPMHImporter job: OAIPMHImporter.finishedJobs.keySet()) jobs.add(job);
+ for (final OAIPMHImporter job: OAIPMHImporter.runningJobs.keySet()) jobs.add(job);
+ for (final OAIPMHImporter job: OAIPMHImporter.startedJobs.keySet()) jobs.add(job);
+ for (final OAIPMHImporter job: OAIPMHImporter.finishedJobs.keySet()) jobs.add(job);
boolean dark = false;
int count = 0;
diff --git a/htroot/IndexImportOAIPMH_p.java b/source/net/yacy/htroot/IndexImportOAIPMH_p.java
similarity index 91%
rename from htroot/IndexImportOAIPMH_p.java
rename to source/net/yacy/htroot/IndexImportOAIPMH_p.java
index 6db80a0c5..df9f8a92c 100644
--- a/htroot/IndexImportOAIPMH_p.java
+++ b/source/net/yacy/htroot/IndexImportOAIPMH_p.java
@@ -22,6 +22,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
@@ -62,7 +64,7 @@ public class IndexImportOAIPMH_p {
DigestURL url = null;
try {
url = new DigestURL(oaipmhurl);
- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
final OAIPMHLoader r = new OAIPMHLoader(sb.loader, url, sb.surrogatesInPath, agent);
final ResumptionToken rt = r.getResumptionToken();
prop.put("import-one", 1);
@@ -97,7 +99,7 @@ public class IndexImportOAIPMH_p {
DigestURL url = null;
try {
url = new DigestURL(oaipmhurl);
- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
final OAIPMHImporter job = new OAIPMHImporter(sb.loader, agent, url);
job.start();
prop.put("status", 1);
@@ -130,7 +132,7 @@ public class IndexImportOAIPMH_p {
// start jobs for the sources
DigestURL url = null;
- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
while (!sourceList.isEmpty()) {
final String oaipmhurl = sourceList.remove(r.nextInt(sourceList.size()));
try {
diff --git a/htroot/IndexImportWarc_p.java b/source/net/yacy/htroot/IndexImportWarc_p.java
similarity index 87%
rename from htroot/IndexImportWarc_p.java
rename to source/net/yacy/htroot/IndexImportWarc_p.java
index d5564bbe0..325454605 100644
--- a/htroot/IndexImportWarc_p.java
+++ b/source/net/yacy/htroot/IndexImportWarc_p.java
@@ -16,6 +16,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
@@ -49,15 +51,15 @@ public class IndexImportWarc_p {
prop.put("import", 0);
if (post != null) {
if (post.containsKey("file") || post.containsKey("url")) {
- String filename = post.get("file");
+ final String filename = post.get("file");
if (filename != null && filename.length() > 0) {
final File sourcefile = new File(filename);
if (sourcefile.exists()) {
try {
- WarcImporter wi = new WarcImporter(sourcefile);
+ final WarcImporter wi = new WarcImporter(sourcefile);
wi.start();
prop.put("import_thread", "started");
- } catch (IOException ex) {
+ } catch (final IOException ex) {
prop.put("import_thread", "Error: file not found [" + filename + "]");
}
prop.put("import", 1);
@@ -67,16 +69,16 @@ public class IndexImportWarc_p {
prop.put("import_thread", "Error: file not found [" + filename + "]");
}
} else {
- String urlstr = post.get("url");
+ final String urlstr = post.get("url");
if (urlstr != null && urlstr.length() > 0) {
try {
- MultiProtocolURL url = new MultiProtocolURL(urlstr);
- WarcImporter wi = new WarcImporter(url);
+ final MultiProtocolURL url = new MultiProtocolURL(urlstr);
+ final WarcImporter wi = new WarcImporter(url);
wi.start();
prop.put("import_thread", "started");
- } catch (MalformedURLException ex) {
+ } catch (final MalformedURLException ex) {
prop.put("import_thread", ex.getMessage());
- } catch (IOException ex) {
+ } catch (final IOException ex) {
prop.put("import_thread", ex.getMessage());
}
prop.put("import", 1);
diff --git a/htroot/IndexReIndexMonitor_p.java b/source/net/yacy/htroot/IndexReIndexMonitor_p.java
similarity index 97%
rename from htroot/IndexReIndexMonitor_p.java
rename to source/net/yacy/htroot/IndexReIndexMonitor_p.java
index 80ed6f0a6..78e40aac1 100644
--- a/htroot/IndexReIndexMonitor_p.java
+++ b/source/net/yacy/htroot/IndexReIndexMonitor_p.java
@@ -17,6 +17,8 @@
* along with this program in the file lgpl21.txt If not, see
* <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
@@ -44,7 +46,7 @@ import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
public class IndexReIndexMonitor_p {
-
+
/** This servlet name, used for identifying recorded API calls */
private static final String SERVLET_NAME = IndexReIndexMonitor_p.class.getSimpleName() + ".html";
@@ -52,7 +54,7 @@ public class IndexReIndexMonitor_p {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
-
+
/* Acquire a transaction token for the next possible POST form submissions */
final String nextTransactionToken = TransactionManager.getTransactionToken(header);
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken);
@@ -64,15 +66,15 @@ public class IndexReIndexMonitor_p {
if (post != null && post.containsKey("reindexnow") && sb.index.fulltext().connectedLocalSolr()) {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
-
+
migration.reindexToschema(sb);
prop.put("querysize", "0");
prop.put("infomessage","reindex job started");
-
+
reidxbt = sb.getThread(ReindexSolrBusyThread.THREAD_NAME); //get new created job for following posts
- }
+ }
}
-
+
if (reidxbt != null) {
prop.put("reindexjobrunning", 1);
prop.put("querysize", reidxbt.getJobCount());
@@ -94,17 +96,17 @@ public class IndexReIndexMonitor_p {
prop.put("reindexjobrunning_fieldlist", 0);
}
}
-
+
if (post != null && post.containsKey("stopreindex")) {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
-
+
sb.terminateThread(ReindexSolrBusyThread.THREAD_NAME, false);
prop.put("infomessage", "reindex job stopped");
prop.put("reindexjobrunning",0);
} else {
prop.put("infomessage", "reindex is running");
- }
+ }
} else {
prop.put("reindexjobrunning", 0);
if (sb.index.fulltext().connectedLocalSolr()) {
@@ -115,29 +117,29 @@ public class IndexReIndexMonitor_p {
prop.putHTML("infomessage", "! reindex works only with embedded Solr index !");
}
}
-
+
processReindexReport(header, sb, prop, reidxbt instanceof ReindexSolrBusyThread ? (ReindexSolrBusyThread)reidxbt : null);
// recrawl job handling
BusyThread recrawlbt = sb.getThread(RecrawlBusyThread.THREAD_NAME);
-
+
String recrawlQuery = RecrawlBusyThread.DEFAULT_QUERY;
boolean inclerrdoc = RecrawlBusyThread.DEFAULT_INCLUDE_FAILED;
boolean deleteOnRecrawl = RecrawlBusyThread.DEFAULT_DELETE_ON_RECRAWL;
// to signal that a setting shall change the form provides a fixed parameter setup=recrawljob, if not present return status only
if (post != null && "recrawljob".equals(post.get("setup"))) { // it's a command to recrawlThread
-
+
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
-
+
if(post.containsKey("recrawlquerytext")) {
recrawlQuery = post.get("recrawlquerytext");
}
-
+
if (post.containsKey("includefailedurls")) {
inclerrdoc = post.getBoolean("includefailedurls");
}
-
+
if (post.containsKey("deleteOnRecrawl")) {
deleteOnRecrawl = post.getBoolean("deleteOnRecrawl");
}
@@ -149,14 +151,14 @@ public class IndexReIndexMonitor_p {
sb.deployThread(RecrawlBusyThread.THREAD_NAME, "ReCrawl", "recrawl existing documents", null,
new RecrawlBusyThread(Switchboard.getSwitchboard(), recrawlQuery, inclerrdoc, deleteOnRecrawl), 1000);
recrawlbt = sb.getThread(RecrawlBusyThread.THREAD_NAME);
-
+
/* store this call as an api call for easy scheduling possibility */
if(sb.tables != null) {
/* We avoid creating a duplicate of any already recorded API call with the same parameters */
final Row lastExecutedCall = WorkTables
.selectLastExecutedApiCall(IndexReIndexMonitor_p.SERVLET_NAME, post, sb);
if (lastExecutedCall != null && !post.containsKey(WorkTables.TABLE_API_COL_APICALL_PK)) {
- byte[] lastExecutedCallPk = lastExecutedCall.getPK();
+ final byte[] lastExecutedCallPk = lastExecutedCall.getPK();
if (lastExecutedCallPk != null) {
post.add(WorkTables.TABLE_API_COL_APICALL_PK, UTF8.String(lastExecutedCallPk));
}
@@ -193,7 +195,7 @@ public class IndexReIndexMonitor_p {
prop.put("recrawljobrunning_simulationResult", 3);
}
}
-
+
if(post.containsKey("recrawlDefaults")) {
recrawlQuery = RecrawlBusyThread.DEFAULT_QUERY;
inclerrdoc = RecrawlBusyThread.DEFAULT_INCLUDE_FAILED;
@@ -217,9 +219,9 @@ public class IndexReIndexMonitor_p {
}
}
}
-
+
processRecrawlReport(header, sb, prop, (RecrawlBusyThread)recrawlbt);
-
+
// just post status of recrawlThread
if (recrawlbt != null && !recrawlbt.shutdownInProgress()) { // provide status
prop.put("recrawljobrunning", 1);
@@ -237,11 +239,11 @@ public class IndexReIndexMonitor_p {
// return rewrite properties
return prop;
}
-
+
/**
* Write information on the eventual currently running or last reindex job
* terminated
- *
+ *
* @param header
* current request header. Must not be null.
* @param sb
@@ -294,10 +296,10 @@ public class IndexReIndexMonitor_p {
final serverObjects prop, final RecrawlBusyThread recrawlbt) {
if (recrawlbt != null) {
prop.put("recrawlReport", 1);
-
+
prop.put("recrawlReport_error", recrawlbt.isTerminatedBySolrFailure());
-
-
+
+
int jobStatus;
if(recrawlbt.isAlive()) {
if(recrawlbt.shutdownInProgress()) {
@@ -309,12 +311,12 @@ public class IndexReIndexMonitor_p {
jobStatus = 2; // Terminated
}
prop.put("recrawlReport_jobStatus", jobStatus);
-
+
prop.put("recrawlReport_recrawlquerytext", recrawlbt.getQuery());
-
+
Locale formatLocale;
if (sb != null) {
- String lng = sb.getConfig("locale.language", Locale.ENGLISH.getLanguage());
+ final String lng = sb.getConfig("locale.language", Locale.ENGLISH.getLanguage());
if ("browser".equals(lng)) {
/* Only use the client locale when locale.language is set to browser */
formatLocale = header.getLocale();
diff --git a/htroot/IndexSchema_p.java b/source/net/yacy/htroot/IndexSchema_p.java
similarity index 84%
rename from htroot/IndexSchema_p.java
rename to source/net/yacy/htroot/IndexSchema_p.java
index c40b1e8c0..ae00ede54 100644
--- a/htroot/IndexSchema_p.java
+++ b/source/net/yacy/htroot/IndexSchema_p.java
@@ -18,6 +18,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
@@ -40,11 +42,11 @@ public class IndexSchema_p {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
-
+
String schemaName = CollectionSchema.CORE_NAME;
- if (post != null) schemaName = post.get("core", schemaName);
- SchemaConfiguration cs = schemaName.equals(CollectionSchema.CORE_NAME) ? sb.index.fulltext().getDefaultConfiguration() : sb.index.fulltext().getWebgraphConfiguration();
-
+ if (post != null) schemaName = post.get("core", schemaName);
+ final SchemaConfiguration cs = schemaName.equals(CollectionSchema.CORE_NAME) ? sb.index.fulltext().getDefaultConfiguration() : sb.index.fulltext().getWebgraphConfiguration();
+
if (post != null && post.containsKey("set")) {
// read index schema table flags
final Iterator<SchemaConfiguration.Entry> i = cs.entryIterator();
@@ -69,7 +71,7 @@ public class IndexSchema_p {
entry.setEnable(c);
modified = true;
}
- }
+ }
}
if (modified) { // save settings to config file if modified
try {
@@ -78,11 +80,11 @@ public class IndexSchema_p {
sb.index.fulltext().getWebgraphConfiguration().commit();
modified = false;
} catch (final IOException ex) {}
- }
+ }
}
-
+
if (post != null && post.containsKey("resetselectiontodefault")) {
- // reset Solr field selection to default configuration
+ // reset Solr field selection to default configuration
File solrInitFile;
if (cs instanceof WebgraphConfiguration) { // get default configuration for webgraph
solrInitFile = new File(sb.getAppPath(), "defaults/" + Switchboard.SOLR_WEBGRAPH_CONFIGURATION_NAME);
@@ -90,10 +92,10 @@ public class IndexSchema_p {
solrInitFile = new File(sb.getAppPath(), "defaults/" + Switchboard.SOLR_COLLECTION_CONFIGURATION_NAME);
}
try {
- SchemaConfiguration solrConfigurationInit = new SchemaConfiguration(solrInitFile);
- Iterator<SchemaConfiguration.Entry> it = cs.entryIterator(); // get current configuration
+ final SchemaConfiguration solrConfigurationInit = new SchemaConfiguration(solrInitFile);
+ final Iterator<SchemaConfiguration.Entry> it = cs.entryIterator(); // get current configuration
while (it.hasNext()) { // iterate over entries and enable/disable according to default
- SchemaConfiguration.Entry etr = it.next();
+ final SchemaConfiguration.Entry etr = it.next();
etr.setEnable(solrConfigurationInit.contains(etr.key()));
}
cs.commit();
@@ -101,26 +103,26 @@ public class IndexSchema_p {
ConcurrentLog.warn("IndexSchema", "file " + solrInitFile.getAbsolutePath() + " not found");
}
}
-
+
/* Acquire a transaction token for the next possible POST IndexReIndeMonitor_p form submission */
prop.put("IndexReIndex_" + TransactionManager.TRANSACTION_TOKEN_PARAM,
TransactionManager.getTransactionToken(header, "/IndexReIndexMonitor_p.html"));
-
+
int c = 0;
boolean dark = false;
// use enum SolrField to keep defined order
- SchemaDeclaration[] cc = schemaName.equals(CollectionSchema.CORE_NAME) ? CollectionSchema.values() : WebgraphSchema.values();
+ final SchemaDeclaration[] cc = schemaName.equals(CollectionSchema.CORE_NAME) ? CollectionSchema.values() : WebgraphSchema.values();
String filterstr = null;
// set active filter button property
- boolean viewall = (post == null) || (filterstr = post.get("filter")) == null;
- boolean viewactiveonly = !viewall && "active".equals(filterstr);
- boolean viewdisabledonly = !viewall && "disabled".equals(filterstr);
+ final boolean viewall = (post == null) || (filterstr = post.get("filter")) == null;
+ final boolean viewactiveonly = !viewall && "active".equals(filterstr);
+ final boolean viewdisabledonly = !viewall && "disabled".equals(filterstr);
prop.put("viewall",viewall);
prop.put("activeonly", viewactiveonly);
prop.put("disabledonly", viewdisabledonly);
-
- for(SchemaDeclaration field : cc) {
- boolean showline = viewactiveonly ? cs.contains(field.name()) : (viewdisabledonly ? cs.containsDisabled(field.name()): true);
+
+ for(final SchemaDeclaration field : cc) {
+ final boolean showline = viewactiveonly ? cs.contains(field.name()) : (viewdisabledonly ? cs.containsDisabled(field.name()): true);
if (showline) {
prop.put("schema_" + c + "_dark", dark ? 1 : 0); dark = !dark;
prop.put("schema_" + c + "_checked", cs.contains(field.name()) ? 1 : 0);
@@ -139,7 +141,7 @@ public class IndexSchema_p {
prop.put("cores_" + 1 + "_selected", WebgraphSchema.CORE_NAME.equals(schemaName) ? 1 : 0);
prop.put("cores", 2);
prop.put("core", schemaName);
-
+
// return rewrite properties
return prop;
}
diff --git a/htroot/IndexShare_p.java b/source/net/yacy/htroot/IndexShare_p.java
similarity index 95%
rename from htroot/IndexShare_p.java
rename to source/net/yacy/htroot/IndexShare_p.java
index 6f17a2749..10f722ec2 100644
--- a/htroot/IndexShare_p.java
+++ b/source/net/yacy/htroot/IndexShare_p.java
@@ -28,6 +28,8 @@
//import java.net.*;
//import java.io.*;
//import de.anomic.tools.*;
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@@ -43,7 +45,7 @@ public class IndexShare_p {
final serverObjects prop = new serverObjects();
// get segment
- Segment indexSegment = sb.index;
+ final Segment indexSegment = sb.index;
if (post == null) {
prop.put("linkfreq", sb.getConfigLong("defaultLinkReceiveFrequency",30));
diff --git a/htroot/Load_MediawikiWiki.java b/source/net/yacy/htroot/Load_MediawikiWiki.java
similarity index 98%
rename from htroot/Load_MediawikiWiki.java
rename to source/net/yacy/htroot/Load_MediawikiWiki.java
index 8af1c8fe1..62f0bc994 100644
--- a/htroot/Load_MediawikiWiki.java
+++ b/source/net/yacy/htroot/Load_MediawikiWiki.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Set;
import net.yacy.cora.protocol.RequestHeader;
@@ -49,7 +51,7 @@ public class Load_MediawikiWiki {
}
prop.put("starturl", "http://");
prop.put("address", myUrl);
-
+
// hidden form param : clean up search events cache ?
if (post != null && post.containsKey("cleanSearchCache")) {
prop.put("cleanSearchCacheChecked", post.getBoolean("cleanSearchCache"));
diff --git a/htroot/Load_PHPBB3.java b/source/net/yacy/htroot/Load_PHPBB3.java
similarity index 99%
rename from htroot/Load_PHPBB3.java
rename to source/net/yacy/htroot/Load_PHPBB3.java
index 899394ae1..3291380d6 100644
--- a/htroot/Load_PHPBB3.java
+++ b/source/net/yacy/htroot/Load_PHPBB3.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Set;
import net.yacy.cora.protocol.RequestHeader;
@@ -50,7 +52,7 @@ public class Load_PHPBB3 {
final boolean intranet = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "").equals("intranet");
prop.put("starturl", (intranet) ? myUrl : "http://");
prop.put("address", myUrl);
-
+
// hidden form param : clean up search events cache ?
if (post != null && post.containsKey("cleanSearchCache")) {
prop.put("cleanSearchCacheChecked", post.getBoolean("cleanSearchCache"));
diff --git a/htroot/Load_RSS_p.java b/source/net/yacy/htroot/Load_RSS_p.java
similarity index 95%
rename from htroot/Load_RSS_p.java
rename to source/net/yacy/htroot/Load_RSS_p.java
index 531368380..05975966c 100644
--- a/htroot/Load_RSS_p.java
+++ b/source/net/yacy/htroot/Load_RSS_p.java
@@ -18,6 +18,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.io.IOException;
import java.net.MalformedURLException;
import java.text.DateFormat;
@@ -72,8 +74,8 @@ public class Load_RSS_p {
final Switchboard sb = (Switchboard)env;
final String collection = post == null ? "user" : CommonPattern.SPACE.matcher(post.get("collection", "user").trim()).replaceAll("");
- Map<String, Pattern> collections = CrawlProfile.collectionParser(collection);
- boolean collectionEnabled = sb.index.fulltext().getDefaultConfiguration().isEmpty() || sb.index.fulltext().getDefaultConfiguration().contains(CollectionSchema.collection_sxt);
+ final Map<String, Pattern> collections = CrawlProfile.collectionParser(collection);
+ final boolean collectionEnabled = sb.index.fulltext().getDefaultConfiguration().isEmpty() || sb.index.fulltext().getDefaultConfiguration().contains(CollectionSchema.collection_sxt);
prop.put("showload_collectionEnabled", collectionEnabled ? 1 : 0);
prop.put("showload_collection", collection);
prop.put("showload", 0);
@@ -165,7 +167,7 @@ public class Load_RSS_p {
}
if (post != null && post.containsKey("addSelectedFeedScheduler")) {
- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
for (final Map.Entry<String, String> entry: post.entrySet()) {
if (entry.getValue().startsWith(CHECKBOX_ITEM_PREFIX)) {
Row row;
@@ -274,7 +276,7 @@ public class Load_RSS_p {
ConcurrentLog.warn("Load_RSS", "url not well-formed: '" + post.get("url", "") + "'");
}
- ClientIdentification.Agent agent = post == null ? ClientIdentification.yacyInternetCrawlerAgent : ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
+ final ClientIdentification.Agent agent = post == null ? ClientIdentification.yacyInternetCrawlerAgent : ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
// if we have an url then try to load the rss
RSSReader rss = null;
@@ -305,7 +307,7 @@ public class Load_RSS_p {
DigestURL messageUrl;
try {
messageUrl = new DigestURL(message.getLink());
- } catch (MalformedURLException e) {
+ } catch (final MalformedURLException e) {
ConcurrentLog.warn("Load_RSS", "Malformed feed item link URL : " + message.getLink());
continue loop;
}
@@ -323,7 +325,7 @@ public class Load_RSS_p {
DigestURL mediaUrl;
try {
mediaUrl = new DigestURL(message.getEnclosure());
- } catch (MalformedURLException e) {
+ } catch (final MalformedURLException e) {
ConcurrentLog.warn("Load_RSS", "Malformed feed item enclosure URL : " + message.getEnclosure());
continue loop;
}
@@ -337,7 +339,7 @@ public class Load_RSS_p {
final List<DigestURL> urlsToIndex = new ArrayList<DigestURL>();
loop: for (final Map.Entry<String, DigestURL> entry: hash2UrlMap.entrySet()) {
final DigestURL messageUrl = entry.getValue();
- HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(messageUrl.hash()));
+ final HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(messageUrl.hash()));
if (harvestProcess != null) {
continue loop;
}
@@ -410,7 +412,7 @@ public class Load_RSS_p {
pubDate = item.getPubDate();
if(link != null && StringUtils.isNotEmpty(item.getGuid())) {
- HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(link.hash()));
+ final HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(link.hash()));
prop.put("showitems_item_" + i + "_hasLink", true);
prop.putHTML("showitems_item_" + i + "_hasLink_link", link.toNormalform(true));
@@ -433,7 +435,7 @@ public class Load_RSS_p {
i++;
if(enclosure != null && enclosure != link && StringUtils.isNotEmpty(item.getGuid())) {
- HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(enclosure.hash()));
+ final HarvestProcess harvestProcess = sb.getHarvestProcess(ASCII.String(enclosure.hash()));
prop.put("showitems_item_" + i + "_hasLink", true);
prop.putHTML("showitems_item_" + i + "_hasLink_link", enclosure.toNormalform(true));
diff --git a/htroot/MessageSend_p.java b/source/net/yacy/htroot/MessageSend_p.java
similarity index 95%
rename from htroot/MessageSend_p.java
rename to source/net/yacy/htroot/MessageSend_p.java
index 57271bc00..9569f778f 100644
--- a/htroot/MessageSend_p.java
+++ b/source/net/yacy/htroot/MessageSend_p.java
@@ -25,6 +25,8 @@
//javac -classpath .:../Classes MessageSend_p.java
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Date;
@@ -78,7 +80,7 @@ public class MessageSend_p {
// open an editor page for the message
// first ask if the other peer is online, and also what kind of document it accepts
- Seed seed = sb.peers.getConnected(ASCII.getBytes(hash));
+ final Seed seed = sb.peers.getConnected(ASCII.getBytes(hash));
if (seed != null) {
for (final String ip : seed.getIPs()) {
Map<String, String> result = null;
@@ -132,7 +134,7 @@ public class MessageSend_p {
sb.peers.peerActions.interfaceDeparture(targetPeer, ip);
}
} else {
-
+
prop.put("mode_permission", "1");
// write input form
@@ -172,7 +174,7 @@ public class MessageSend_p {
if (subject.length() > 100) subject = subject.substring(0, 100);
if (message.length() > messagesize) message = message.substring(0, messagesize);
final byte[] mb = UTF8.getBytes(message);
- SeedDB seedDB = sb.peers;
+ final SeedDB seedDB = sb.peers;
// prepare request
final String salt = crypt.randomSalt();
@@ -183,7 +185,7 @@ public class MessageSend_p {
parts.put("subject", UTF8.StringBody(subject));
parts.put("message", UTF8.StringBody(mb));
final Seed seed = seedDB.getConnected(ASCII.getBytes(hash));
- boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
+ final boolean preferHttps = sb.getConfigBool(SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED,
SwitchboardConstants.NETWORK_PROTOCOL_HTTPS_PREFERRED_DEFAULT);
Post post1 = null;
for(final String ip : seed.getIPs()) {
@@ -209,7 +211,7 @@ public class MessageSend_p {
}
}
}
-
+
if (post1 != null) {
break;
}
diff --git a/htroot/Messages_p.java b/source/net/yacy/htroot/Messages_p.java
similarity index 97%
rename from htroot/Messages_p.java
rename to source/net/yacy/htroot/Messages_p.java
index 40540d281..26d5f513b 100644
--- a/htroot/Messages_p.java
+++ b/source/net/yacy/htroot/Messages_p.java
@@ -24,12 +24,16 @@
// javac -classpath .:../Classes Message.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.util.Date;
import java.util.Iterator;
import java.util.TreeMap;
+import com.google.common.io.Files;
+
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.protocol.HeaderFramework;
@@ -42,8 +46,6 @@ import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
-import com.google.common.io.Files;
-
public class Messages_p {
diff --git a/htroot/Network.java b/source/net/yacy/htroot/Network.java
similarity index 95%
rename from htroot/Network.java
rename to source/net/yacy/htroot/Network.java
index 5e993fc68..64519f686 100644
--- a/htroot/Network.java
+++ b/source/net/yacy/htroot/Network.java
@@ -27,6 +27,8 @@
// javac -classpath .:../classes Network.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.HashSet;
@@ -68,11 +70,11 @@ public class Network {
final long start = System.currentTimeMillis();
final servletProperties prop = new servletProperties();
-
+
prop.put("menu", post == null ? 2 : (post.get("menu", "").equals("embed")) ? 0 : (post.get("menu","").equals("simple")) ? 1 : 2);
if (sb.peers.mySeed() != null) prop.put("menu_newpeer_peerhash", sb.peers.mySeed().hash);
- String pathInfo = requestHeader.getPathInfo();
+ final String pathInfo = requestHeader.getPathInfo();
prop.setLocalized(!pathInfo.endsWith(".xml") && !pathInfo.endsWith(".json"));
prop.putHTML("page_networkTitle", sb.getConfig("network.unit.description", "unspecified"));
prop.putHTML("page_networkName", sb.getConfig(SwitchboardConstants.NETWORK_NAME, "unspecified"));
@@ -128,10 +130,10 @@ public class Network {
accActLinks += LCount;
accActWords += ICount;
}
- String port = seed.get(Seed.PORT, "-");
- Set<String> ips = seed.getIPs();
+ final String port = seed.get(Seed.PORT, "-");
+ final Set<String> ips = seed.getIPs();
int ipsc = 0;
- for (String s: ips) {
+ for (final String s: ips) {
prop.put("table_ips_" + ipsc + "_nodestate", seed.getFlagRootNode() ? 1 : 0);
prop.put("table_ips_" + ipsc + "_c", 0);
prop.putHTML("table_ips_" + ipsc + "_c_hash", seed.hash);
@@ -221,27 +223,27 @@ public class Network {
}
final ConcurrentMap<String, String> map = new ConcurrentHashMap<String, String>();
- String challengeIP = post.get("peerIP");
- String challengePort = post.get("peerPort");
+ final String challengeIP = post.get("peerIP");
+ final String challengePort = post.get("peerPort");
map.put(Seed.IP, challengeIP);
map.put(Seed.PORT, challengePort);
Seed peer = post.get("peerHash") == null ? null : new Seed(post.get("peerHash"), map);
sb.updateMySeed();
- Seed mySeed = sb.peers.mySeed();
+ final Seed mySeed = sb.peers.mySeed();
final String challengeURLStr = peer.getPublicURL(challengeIP, false);
try {
final MultiProtocolURL challengeURL = new MultiProtocolURL(challengeURLStr);
final Map<String, String> response = Protocol.hello(mySeed, sb.peers.peerActions, challengeURL, peer.hash);
-
+
if (response == null) {
- Seed peerd = sb.peers.get(peer.hash);
+ final Seed peerd = sb.peers.get(peer.hash);
if (peerd != null) peer = peerd;
sb.peers.peerActions.interfaceDeparture(peer, challengeIP);
prop.put("table_comment",1);
prop.put("table_comment_status", "publish: no response from peer '" + peer.getName() + "/" + post.get("peerHash") + "' from <a href=\"" + challengeURL + "\" target=\"_blank\">" + challengeURL + "</a>");
} else {
- String yourtype = response.get("yourtype");
- String yourip = response.get("yourip");
+ final String yourtype = response.get("yourtype");
+ final String yourip = response.get("yourip");
peer = sb.peers.getConnected(peer.hash);
if (peer == null) {
prop.put("table_comment",1);
@@ -436,11 +438,11 @@ public class Network {
}
if (location.length() > 10) location = location.substring(0, 10);
if (location.length() == 0) {
- Locale l = Domains.getLocale(ip);
+ final Locale l = Domains.getLocale(ip);
if (l != null) location = l.toString();
}
prop.putHTML(STR_TABLE_LIST + conCount + "_location", location);
- String port = seed.get(Seed.PORT, "-");
+ final String port = seed.get(Seed.PORT, "-");
int ipsc = 0;
for (final String s: ips) {
prop.put(STR_TABLE_LIST + conCount + "_ips_" + ipsc + "_nodestate", seed.getFlagRootNode() ? 1 : 0);
@@ -506,7 +508,7 @@ public class Network {
final String peertags = MapTools.set2string(seed.getPeerTags(), ",", false);
prop.putHTML(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", ((peertags == null) || (peertags.isEmpty())) ? "no tags given" : ("tags = " + peertags));
}
- String[] yv = yacyVersion.combined2prettyVersion(seed.get(Seed.VERSION, "0.1"), shortname);
+ final String[] yv = yacyVersion.combined2prettyVersion(seed.get(Seed.VERSION, "0.1"), shortname);
prop.putHTML(STR_TABLE_LIST + conCount + "_version", yv[0] + "/" + yv[1]);
prop.putNum(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen);
prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(Seed.UTC, "-"));
@@ -550,9 +552,9 @@ public class Network {
if(path != null && path.endsWith(".xml")) {
final ResponseHeader outgoingHeader = new ResponseHeader(200);
outgoingHeader.put(HeaderFramework.CORS_ALLOW_ORIGIN, "*");
- prop.setOutgoingHeader(outgoingHeader);
+ prop.setOutgoingHeader(outgoingHeader);
}
-
+
// return rewrite properties
return prop;
}
diff --git a/htroot/NetworkHistory.java b/source/net/yacy/htroot/NetworkHistory.java
similarity index 85%
rename from htroot/NetworkHistory.java
rename to source/net/yacy/htroot/NetworkHistory.java
index da7833f97..043ef3b7f 100644
--- a/htroot/NetworkHistory.java
+++ b/source/net/yacy/htroot/NetworkHistory.java
@@ -18,6 +18,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
@@ -42,7 +44,7 @@ import net.yacy.visualization.RasterPlotter;
public class NetworkHistory {
-
+
public static RasterPlotter respond(@SuppressWarnings("unused") final RequestHeader header, serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
@@ -63,22 +65,22 @@ public class NetworkHistory {
cI size of the index (number of documents)
*/
final Set<String> columns = new LinkedHashSet<>();
- for (String col: columnsx) columns.add(col);
+ for (final String col: columnsx) columns.add(col);
// scan the database and put in values
- List<Map<String, Long>> rows = new ArrayList<>(maxtime * 2);
- long now = System.currentTimeMillis();
- long timelimit = now - maxtime * 3600000L;
+ final List<Map<String, Long>> rows = new ArrayList<>(maxtime * 2);
+ final long now = System.currentTimeMillis();
+ final long timelimit = now - maxtime * 3600000L;
try {
- Iterator<Row> rowi = sb.tables.iterator("stats", false);
+ final Iterator<Row> rowi = sb.tables.iterator("stats", false);
Map <String, Long> statrow;
while (rowi.hasNext()) {
- Row row = rowi.next();
- String d = ASCII.String(row.getPK());
- Date date = GenericFormatter.SHORT_MINUTE_FORMATTER.parse(d, 0).getTime();
+ final Row row = rowi.next();
+ final String d = ASCII.String(row.getPK());
+ final Date date = GenericFormatter.SHORT_MINUTE_FORMATTER.parse(d, 0).getTime();
if (date.getTime() < timelimit) break;
statrow = new HashMap<>();
- for (String key: columns) {
- byte[] x = row.get(key);
+ for (final String key: columns) {
+ final byte[] x = row.get(key);
if (x != null) statrow.put(key, Long.parseLong(ASCII.String(x)));
}
statrow.put("time", date.getTime());
@@ -87,12 +89,12 @@ public class NetworkHistory {
} catch (final IOException|ParseException e) {
ConcurrentLog.logException(e);
}
-
+
// find correct scale
int maxpeers = 100, minpeers = Integer.MAX_VALUE; // to be measured by pre-scanning the db
- for (Map<String, Long> row: rows) {
- for (String column: columns) {
- Long v = row.get(column);
+ for (final Map<String, Long> row: rows) {
+ for (final String column: columns) {
+ final Long v = row.get(column);
if (v != null) maxpeers = Math.max(maxpeers, (int) v.longValue());
if (v != null && v.longValue() > 0) minpeers = Math.min(minpeers, (int) v.longValue());
}
@@ -132,24 +134,24 @@ public class NetworkHistory {
if (columns.contains("cP")) headline += ", POTENTIAL JUNIOR PEERS";
if (columns.contains("cI")) headline = "YACY PEER '" + sb.peers.myName().toUpperCase() + "' INDEX SIZE HISTORY: NUMBER OF DOCUMENTS";
if (columns.contains("cR")) headline = "YACY PEER '" + sb.peers.myName().toUpperCase() + "' INDEX SIZE HISTORY: NUMBER OF RWI ENTRIES";
- ChartPlotter chart = new ChartPlotter(width, height, 0xFFFFFFl, 0x000000l, 0xAAAAAAl, leftborder, rightborder, topborder, bottomborder, headline, "IN THE LAST " + timestr);
+ final ChartPlotter chart = new ChartPlotter(width, height, 0xFFFFFFl, 0x000000l, 0xAAAAAAl, leftborder, rightborder, topborder, bottomborder, headline, "IN THE LAST " + timestr);
long pps = (long)hspace * (long)bottomscale / maxtime;
int pixelperscale = Math.max(8, (int)pps );
chart.declareDimension(ChartPlotter.DIMENSION_BOTTOM, bottomscale, pixelperscale, -maxtime, 0x000000l, 0xCCCCCCl, "TIME/HOURS");
pps = (long)vspace * (long)leftscale / (maxpeers-minpeers);
pixelperscale = Math.max(8, (int)pps );
chart.declareDimension(ChartPlotter.DIMENSION_LEFT, leftscale, pixelperscale, minpeers, 0x008800l, null , columns.contains("cI") ? "DOCUMENTS" : columns.contains("cR") ? "RWIs" : "PEERS");
-
+
// write the data
float x0, x1;
int y0, y1;
Long time;
- for (String column: columns) {
+ for (final String column: columns) {
x0 = 1.0f; y0 = 0;
- for (Map<String, Long> row: rows) {
+ for (final Map<String, Long> row: rows) {
time = row.get("time");
if (time == null) continue;
- Long v = row.get(column);
+ final Long v = row.get(column);
if (v == null) continue;
x1 = (time - now) / 3600000.0f;
y1 = (int) v.longValue();
diff --git a/htroot/NetworkPicture.java b/source/net/yacy/htroot/NetworkPicture.java
similarity index 96%
rename from htroot/NetworkPicture.java
rename to source/net/yacy/htroot/NetworkPicture.java
index 51b8b628d..52b177f43 100644
--- a/htroot/NetworkPicture.java
+++ b/source/net/yacy/htroot/NetworkPicture.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.concurrent.Semaphore;
import net.yacy.cora.protocol.RequestHeader;
@@ -41,7 +43,7 @@ public class NetworkPicture {
private static final ConcurrentLog log = new ConcurrentLog("NetworkPicture");
private static final Semaphore sync = new Semaphore(1, true);
private static long lastAccessSeconds = 0;
-
+
public static EncodedImage respond(
final RequestHeader header,
final serverObjects post,
diff --git a/htroot/News.java b/source/net/yacy/htroot/News.java
similarity index 96%
rename from htroot/News.java
rename to source/net/yacy/htroot/News.java
index 92a288958..296e75cb0 100644
--- a/htroot/News.java
+++ b/source/net/yacy/htroot/News.java
@@ -24,10 +24,12 @@
// javac -classpath .:../classes Network.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.Iterator;
import java.util.Map;
-import java.util.Set;
import java.util.Map.Entry;
+import java.util.Set;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.protocol.HeaderFramework;
@@ -162,9 +164,9 @@ public class News {
if(seed == null) {
link = "";
} else {
- Set<String> ips = seed.getIPs();
+ final Set<String> ips = seed.getIPs();
if(!ips.isEmpty()) {
- link = "http://" + seed.getPublicAddress(ips.iterator().next()) + "/Wiki.html?page=" + record.attribute("page", "");
+ link = "http://" + seed.getPublicAddress(ips.iterator().next()) + "/Wiki.html?page=" + record.attribute("page", "");
} else {
link = "";
}
@@ -175,7 +177,7 @@ public class News {
if(seed == null) {
link = "";
} else {
- Set<String> ips = seed.getIPs();
+ final Set<String> ips = seed.getIPs();
if(!ips.isEmpty()) {
link = "http://" + seed.getPublicAddress(ips.iterator().next()) + "/Blog.html?page=" + record.attribute("page", "");
} else {
diff --git a/htroot/PeerLoadPicture.java b/source/net/yacy/htroot/PeerLoadPicture.java
similarity index 96%
rename from htroot/PeerLoadPicture.java
rename to source/net/yacy/htroot/PeerLoadPicture.java
index 3482279cf..2f014da47 100644
--- a/htroot/PeerLoadPicture.java
+++ b/source/net/yacy/htroot/PeerLoadPicture.java
@@ -1,3 +1,5 @@
+package net.yacy.htroot;
+
import java.awt.Color;
import java.awt.Image;
import java.util.HashMap;
diff --git a/htroot/PerformanceConcurrency_p.java b/source/net/yacy/htroot/PerformanceConcurrency_p.java
similarity index 99%
rename from htroot/PerformanceConcurrency_p.java
rename to source/net/yacy/htroot/PerformanceConcurrency_p.java
index 051a207e5..05cf21b8d 100644
--- a/htroot/PerformanceConcurrency_p.java
+++ b/source/net/yacy/htroot/PerformanceConcurrency_p.java
@@ -19,6 +19,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Iterator;
import net.yacy.cora.protocol.RequestHeader;
diff --git a/htroot/PerformanceGraph.java b/source/net/yacy/htroot/PerformanceGraph.java
similarity index 94%
rename from htroot/PerformanceGraph.java
rename to source/net/yacy/htroot/PerformanceGraph.java
index bb1c45b9a..dc8cc7224 100644
--- a/htroot/PerformanceGraph.java
+++ b/source/net/yacy/htroot/PerformanceGraph.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.concurrent.TimeUnit;
import net.yacy.cora.protocol.RequestHeader;
@@ -37,7 +39,7 @@ public class PerformanceGraph {
private static long indeSizeCache = 0;
private static long indexSizeTime = 0;
-
+
public static RasterPlotter respond(@SuppressWarnings("unused") final RequestHeader header, serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
@@ -48,12 +50,12 @@ public class PerformanceGraph {
final boolean showMemory = !post.containsKey("nomem");
final boolean showPeers = !post.containsKey("nopeers");
- long t = System.currentTimeMillis();
+ final long t = System.currentTimeMillis();
if (t - indexSizeTime > 10000) {
indeSizeCache = sb.index.fulltext().collectionSize();
indexSizeTime = t;
}
- RasterPlotter graph = ProfilingGraph.performanceGraph(
+ final RasterPlotter graph = ProfilingGraph.performanceGraph(
width, height,
indeSizeCache + " URLS / " + sb.index.RWICount() + " WORDS IN INDEX / " + sb.index.RWIBufferCount() + " WORDS IN CACHE",
600, TimeUnit.SECONDS,
diff --git a/htroot/PerformanceMemory_p.java b/source/net/yacy/htroot/PerformanceMemory_p.java
similarity index 96%
rename from htroot/PerformanceMemory_p.java
rename to source/net/yacy/htroot/PerformanceMemory_p.java
index 37710a4a3..4cae9ea15 100644
--- a/htroot/PerformanceMemory_p.java
+++ b/source/net/yacy/htroot/PerformanceMemory_p.java
@@ -25,6 +25,8 @@
//if the shell's current path is HTROOT
//import java.util.Iterator;
+package net.yacy.htroot;
+
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.Map;
@@ -59,7 +61,7 @@ public class PerformanceMemory_p {
prop.put("gc", "1");
prop.put("autoreload.checked", "1");
} else if (post.containsKey("dummy")) {
- boolean simulatedshortmemory = post.containsKey("simulatedshortmemory");
+ final boolean simulatedshortmemory = post.containsKey("simulatedshortmemory");
MemoryControl.setSimulatedShortStatus(simulatedshortmemory);
if (simulatedshortmemory) prop.put("autoreload.checked", "1");
final boolean std = post.containsKey("useStandardmemoryStrategy");
@@ -98,7 +100,7 @@ public class PerformanceMemory_p {
prop.putNum("memoryUsedNow", MemoryControl.used() / MB);
// write table for Table index sizes
- Iterator<String> i = Table.filenames();
+ final Iterator<String> i = Table.filenames();
String filename;
TableStatistics stats;
int p;
@@ -198,7 +200,7 @@ public class PerformanceMemory_p {
// prop.putNum("objectCacheStartShrink", Cache.getMemStartShrink() / (1024d * 1024d));
// prop.putNum("objectHitCacheTotalMem", totalhitmem / (1024d * 1024d));
// prop.putNum("objectMissCacheTotalMem", totalmissmem / (1024d * 1024d));
-
+
prop.putNum("namecacheHit.size", Domains.nameCacheHitSize());
prop.putNum("namecacheHit.Hit", Domains.cacheHit_Hit);
prop.putNum("namecacheHit.Miss", Domains.cacheHit_Miss);
diff --git a/htroot/PerformanceQueues_p.java b/source/net/yacy/htroot/PerformanceQueues_p.java
similarity index 99%
rename from htroot/PerformanceQueues_p.java
rename to source/net/yacy/htroot/PerformanceQueues_p.java
index b728216a2..6b274edce 100644
--- a/htroot/PerformanceQueues_p.java
+++ b/source/net/yacy/htroot/PerformanceQueues_p.java
@@ -24,6 +24,8 @@
//javac -classpath .:../classes Network.java
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import java.util.Iterator;
import java.util.Map;
diff --git a/htroot/PerformanceSearch_p.java b/source/net/yacy/htroot/PerformanceSearch_p.java
similarity index 96%
rename from htroot/PerformanceSearch_p.java
rename to source/net/yacy/htroot/PerformanceSearch_p.java
index 35230934f..001fe00d7 100644
--- a/htroot/PerformanceSearch_p.java
+++ b/source/net/yacy/htroot/PerformanceSearch_p.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Iterator;
import net.yacy.cora.protocol.RequestHeader;
diff --git a/htroot/Performance_p.java b/source/net/yacy/htroot/Performance_p.java
similarity index 95%
rename from htroot/Performance_p.java
rename to source/net/yacy/htroot/Performance_p.java
index 91f5d1ea7..cb220f0ff 100644
--- a/htroot/Performance_p.java
+++ b/source/net/yacy/htroot/Performance_p.java
@@ -24,12 +24,14 @@
//javac -classpath .:../classes Network.java
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
public class Performance_p {
-
+
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch sb) {
return PerformanceQueues_p.respond(header, post, sb);
}
diff --git a/htroot/ProxyIndexingMonitor_p.java b/source/net/yacy/htroot/ProxyIndexingMonitor_p.java
similarity index 97%
rename from htroot/ProxyIndexingMonitor_p.java
rename to source/net/yacy/htroot/ProxyIndexingMonitor_p.java
index 75704192b..92a35432d 100644
--- a/htroot/ProxyIndexingMonitor_p.java
+++ b/source/net/yacy/htroot/ProxyIndexingMonitor_p.java
@@ -27,6 +27,8 @@
// javac -classpath .:../classes ProxyIndexingMonitor_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import net.yacy.cora.protocol.RequestHeader;
diff --git a/htroot/QuickCrawlLink_p.java b/source/net/yacy/htroot/QuickCrawlLink_p.java
similarity index 96%
rename from htroot/QuickCrawlLink_p.java
rename to source/net/yacy/htroot/QuickCrawlLink_p.java
index e7c666541..adf0497d1 100644
--- a/htroot/QuickCrawlLink_p.java
+++ b/source/net/yacy/htroot/QuickCrawlLink_p.java
@@ -29,6 +29,8 @@
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.net.MalformedURLException;
import java.util.Date;
@@ -70,7 +72,7 @@ public class QuickCrawlLink_p {
} else {
port = sb.getConfigInt(SwitchboardConstants.SERVER_PORT, 8090);
}
-
+
prop.put("mode_host", Domains.LOCALHOST);
prop.put("mode_port", port);
@@ -88,7 +90,7 @@ public class QuickCrawlLink_p {
crawlingStart = UTF8.decodeURL(crawlingStart);
// get segment
- Segment indexSegment = sb.index;
+ final Segment indexSegment = sb.index;
// get the browser title
String title = post.get("title", null);
@@ -130,8 +132,8 @@ public class QuickCrawlLink_p {
final byte[] urlhash = crawlingStartURL.hash();
indexSegment.fulltext().remove(urlhash);
sb.crawlQueues.noticeURL.removeByURLHash(urlhash);
- int timezoneOffset = post.getInt("timezoneOffset", 0);
-
+ final int timezoneOffset = post.getInt("timezoneOffset", 0);
+
// create crawling profile
CrawlProfile pe = null;
try {
diff --git a/htroot/RankingRWI_p.java b/source/net/yacy/htroot/RankingRWI_p.java
similarity index 99%
rename from htroot/RankingRWI_p.java
rename to source/net/yacy/htroot/RankingRWI_p.java
index 5fba47dd6..9d674e389 100644
--- a/htroot/RankingRWI_p.java
+++ b/source/net/yacy/htroot/RankingRWI_p.java
@@ -25,6 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -73,7 +75,7 @@ public class RankingRWI_p {
rankingParameters.put(RankingProfile.WORDDISTANCE, "Word Distance;a higher ranking level prefers documents where the search words appear close together. This ranking parameter works like a NEAR operator in more-than-one word searches.");
rankingParameters.put(RankingProfile.WORDSINTEXT, "Words In Text;a higher ranking level prefers documents with a large number of words. Be aware that this is a compensation of the term frequency parameter.");
rankingParameters.put(RankingProfile.WORDSINTITLE, "Words In Title;a higher ranking level prefers documents with a large number of words in the document title.");
-
+
rankingParameters.put(RankingProfile.URLCOMPINTOPLIST, "URL Component Appears In Toplist;a higher ranking level prefers documents with words in the url path that match words in the toplist. The toplist is generated dynamically from the search results using a statistic of the most used words. The toplist is a top-10 list of the most used words in URLs and document titles.");
rankingParameters.put(RankingProfile.DESCRCOMPINTOPLIST, "Description Comp. Appears In Toplist;a higher ranking level prefers documents with words in the document description that match words in the toplist. The toplist is generated dynamically from the search results using a statistic of the most used words. The toplist is a top-10 list of the most used words in URLs and document titles.");
rankingParameters.put(RankingProfile.PREFER, "Application Of Prefer Pattern;a higher ranking level prefers documents where the url matches the prefer pattern given in a search request.");
diff --git a/htroot/RankingSolr_p.java b/source/net/yacy/htroot/RankingSolr_p.java
similarity index 77%
rename from htroot/RankingSolr_p.java
rename to source/net/yacy/htroot/RankingSolr_p.java
index 9905ff2e6..f94480e11 100644
--- a/htroot/RankingSolr_p.java
+++ b/source/net/yacy/htroot/RankingSolr_p.java
@@ -18,14 +18,19 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.TreeMap;
import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.DisMaxParams;
import net.yacy.cora.federate.solr.Ranking;
import net.yacy.cora.federate.solr.SchemaDeclaration;
@@ -41,9 +46,6 @@ import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.DisMaxParams;
-
public class RankingSolr_p {
@@ -56,18 +58,18 @@ public class RankingSolr_p {
int profileNr = 0;
if (post != null) profileNr = post.getInt("profileNr", profileNr);
-
+
if (post != null && post.containsKey("EnterBoosts")) {
- StringBuilder boostString = new StringBuilder(); // SwitchboardConstants.SEARCH_RANKING_SOLR_BOOST;
- for (Map.Entry<String, String> entry: post.entrySet()) {
+ final StringBuilder boostString = new StringBuilder(); // SwitchboardConstants.SEARCH_RANKING_SOLR_BOOST;
+ for (final Map.Entry<String, String> entry: post.entrySet()) {
if (entry.getKey().startsWith("boost")) {
- String fieldName = entry.getKey().substring(6);
- CollectionSchema field = CollectionSchema.valueOf(fieldName);
+ final String fieldName = entry.getKey().substring(6);
+ final CollectionSchema field = CollectionSchema.valueOf(fieldName);
if (field == null) continue;
- String fieldValue = entry.getValue();
+ final String fieldValue = entry.getValue();
if (fieldValue == null || fieldValue.length() == 0) continue;
try {
- float boost = Float.parseFloat(fieldValue);
+ final float boost = Float.parseFloat(fieldValue);
if (boost > 0.0f) { // don't allow <= 0
if (boostString.length() > 0) boostString.append(',');
boostString.append(field.getSolrFieldName()).append('^').append(Float.toString(boost));
@@ -78,26 +80,26 @@ public class RankingSolr_p {
}
}
if (boostString.length() > 0) {
- String s = boostString.toString();
+ final String s = boostString.toString();
sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTFIELDS_ + profileNr, s);
sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).updateBoosts(s);
}
}
if (post != null && post.containsKey("ResetBoosts")) {
- String s = "url_paths_sxt^3.0,synonyms_sxt^0.5,title^5.0,text_t^1.0,host_s^6.0,h1_txt^5.0,url_file_name_tokens_t^4.0,h2_txt^3.0,keywords^2.0,description_txt^1.5,author^1.0";
+ final String s = "url_paths_sxt^3.0,synonyms_sxt^0.5,title^5.0,text_t^1.0,host_s^6.0,h1_txt^5.0,url_file_name_tokens_t^4.0,h2_txt^3.0,keywords^2.0,description_txt^1.5,author^1.0";
sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTFIELDS_ + profileNr, s);
sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).updateBoosts(s);
}
if (post != null && post.containsKey("EnterBQ")) {
- String bq = post.get(DisMaxParams.BQ);
+ final String bq = post.get(DisMaxParams.BQ);
if (bq != null) {
sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTQUERY_ + profileNr, bq);
sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setBoostQuery(bq);
}
}
if (post != null && post.containsKey("ResetBQ")) {
- String bq = "crawldepth_i:0^0.8\ncrawldepth_i:1^0.4";
+ final String bq = "crawldepth_i:0^0.8\ncrawldepth_i:1^0.4";
if (bq != null) {
sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTQUERY_ + profileNr, bq);
sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setBoostQuery(bq);
@@ -105,14 +107,14 @@ public class RankingSolr_p {
}
if (post != null && post.containsKey("EnterFQ")) {
- String fq = post.get(CommonParams.FQ);
+ final String fq = post.get(CommonParams.FQ);
if (fq != null) {
sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_FILTERQUERY_ + profileNr, fq);
sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setFilterQuery(fq);
}
}
if (post != null && post.containsKey("ResetFQ")) {
- String fq = ""; // i.e. "http_unique_b:true AND www_unique_b:true"
+ final String fq = ""; // i.e. "http_unique_b:true AND www_unique_b:true"
if (fq != null) {
sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_FILTERQUERY_ + profileNr, fq);
sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setFilterQuery(fq);
@@ -120,14 +122,14 @@ public class RankingSolr_p {
}
if (post != null && post.containsKey("EnterBF")) {
- String bf = post.get(DisMaxParams.BF);
+ final String bf = post.get(DisMaxParams.BF);
if (bf != null) {
sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTFUNCTION_ + profileNr, bf);
sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setBoostFunction(bf);
}
}
if (post != null && post.containsKey("ResetBF")) {
- String bf = "";
+ final String bf = "";
if (bf != null) {
sb.setConfig(SwitchboardConstants.SEARCH_RANKING_SOLR_COLLECTION_BOOSTFUNCTION_ + profileNr, bf);
sb.index.fulltext().getDefaultConfiguration().getRanking(profileNr).setBoostFunction(bf);
@@ -137,11 +139,11 @@ public class RankingSolr_p {
final serverObjects prop = new serverObjects();
int i = 0;
- CollectionConfiguration colcfg = sb.index.fulltext().getDefaultConfiguration();
- Ranking ranking = colcfg.getRanking(profileNr);
- for (SchemaDeclaration field: CollectionSchema.values()) {
+ final CollectionConfiguration colcfg = sb.index.fulltext().getDefaultConfiguration();
+ final Ranking ranking = colcfg.getRanking(profileNr);
+ for (final SchemaDeclaration field: CollectionSchema.values()) {
if (!field.isSearchable()) continue;
- Float boost = ranking.getFieldBoost(field);
+ final Float boost = ranking.getFieldBoost(field);
if (boost != null || colcfg.contains(field)) { // show only available or configured boost fields
prop.put("boosts_" + i + "_field", field.getSolrFieldName());
if (boost == null || boost.floatValue() <= 0.0f) {
@@ -169,12 +171,12 @@ public class RankingSolr_p {
}
prop.put("profiles", 4);
prop.put("profileNr", profileNr);
-
+
// make boost hints for vocabularies
Map<String, ReversibleScoreMap<String>> vocabularyFacet;
try {
vocabularyFacet = sb.index.fulltext().getDefaultConnector().getFacets(CollectionSchema.vocabularies_sxt.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM, 100, CollectionSchema.vocabularies_sxt.getSolrFieldName());
- } catch (IOException e) {
+ } catch (final IOException e) {
ConcurrentLog.logException(e);
vocabularyFacet = new HashMap<>();
}
@@ -183,14 +185,14 @@ public class RankingSolr_p {
} else {
prop.put("boosthint", 1);
prop.putHTML("boosthint_vocabulariesfield", CollectionSchema.vocabularies_sxt.getSolrFieldName());
- ReversibleScoreMap<String> vokcounts = vocabularyFacet.values().iterator().next();
- Collection<String> vocnames = vokcounts.keyList(true);
+ final ReversibleScoreMap<String> vokcounts = vocabularyFacet.values().iterator().next();
+ final Collection<String> vocnames = vokcounts.keyList(true);
prop.putHTML("boosthint_vocabulariesavailable", vocnames.toString());
- ArrayList<String> voccountFields = new ArrayList<>();
- ArrayList<String> voclogcountFields = new ArrayList<>();
- ArrayList<String> voclogcountsFields = new ArrayList<>();
- ArrayList<String> ff = new ArrayList<>();
- for (String vocname: vocnames) {
+ final ArrayList<String> voccountFields = new ArrayList<>();
+ final ArrayList<String> voclogcountFields = new ArrayList<>();
+ final ArrayList<String> voclogcountsFields = new ArrayList<>();
+ final ArrayList<String> ff = new ArrayList<>();
+ for (final String vocname: vocnames) {
voccountFields.add(CollectionSchema.VOCABULARY_PREFIX + vocname + CollectionSchema.VOCABULARY_COUNT_SUFFIX);
voclogcountFields.add(CollectionSchema.VOCABULARY_PREFIX + vocname + CollectionSchema.VOCABULARY_LOGCOUNT_SUFFIX);
voclogcountsFields.add(CollectionSchema.VOCABULARY_PREFIX + vocname + CollectionSchema.VOCABULARY_LOGCOUNTS_SUFFIX);
@@ -200,18 +202,18 @@ public class RankingSolr_p {
prop.putHTML("boosthint_vocabulariesvoccount", voccountFields.toString());
prop.putHTML("boosthint_vocabulariesvoclogcount", voclogcountFields.toString());
prop.putHTML("boosthint_vocabulariesvoclogcounts", voclogcountsFields.toString());
- String[] facetfields = ff.toArray(new String[ff.size()]);
+ final String[] facetfields = ff.toArray(new String[ff.size()]);
int fc = 0;
if (facetfields.length > 0) try {
- LinkedHashMap<String, ReversibleScoreMap<String>> facets = sb.index.fulltext().getDefaultConnector().getFacets("*:*", 100, facetfields);
+ final LinkedHashMap<String, ReversibleScoreMap<String>> facets = sb.index.fulltext().getDefaultConnector().getFacets("*:*", 100, facetfields);
facets.put(CollectionSchema.vocabularies_sxt.getSolrFieldName(), vokcounts);
- for (Map.Entry<String, ReversibleScoreMap<String>> facetentry: facets.entrySet()) {
- ReversibleScoreMap<String> facetfieldmap = facetentry.getValue();
+ for (final Map.Entry<String, ReversibleScoreMap<String>> facetentry: facets.entrySet()) {
+ final ReversibleScoreMap<String> facetfieldmap = facetentry.getValue();
if (facetfieldmap.size() == 0) continue;
- TreeMap<String, Integer> statMap = new TreeMap<>();
- for (String k: facetfieldmap) statMap.put(k, facetfieldmap.get(k));
+ final TreeMap<String, Integer> statMap = new TreeMap<>();
+ for (final String k: facetfieldmap) statMap.put(k, facetfieldmap.get(k));
prop.put("boosthint_facets_" + fc + "_facetname", facetentry.getKey());
- int c = 0; for (Entry<String, Integer> entry: statMap.entrySet()) {
+ int c = 0; for (final Entry<String, Integer> entry: statMap.entrySet()) {
prop.put("boosthint_facets_" + fc + "_facet_" + c + "_key", entry.getKey());
prop.put("boosthint_facets_" + fc + "_facet_" + c + "_count", entry.getValue());
c++;
@@ -219,11 +221,11 @@ public class RankingSolr_p {
prop.put("boosthint_facets_" + fc + "_facet", c);
fc++;
}
- } catch (IOException e) {
+ } catch (final IOException e) {
}
prop.put("boosthint_facets", fc);
}
-
+
return prop;
}
diff --git a/htroot/RegexTest.java b/source/net/yacy/htroot/RegexTest.java
similarity index 90%
rename from htroot/RegexTest.java
rename to source/net/yacy/htroot/RegexTest.java
index 6902e93af..c7a563337 100644
--- a/htroot/RegexTest.java
+++ b/source/net/yacy/htroot/RegexTest.java
@@ -7,17 +7,19 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
- *
+ *
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
- *
+ *
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.util.regex.PatternSyntaxException;
import net.yacy.cora.protocol.RequestHeader;
@@ -25,11 +27,11 @@ import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
public class RegexTest {
-
+
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
- String text = (post == null) ? "" : post.get("text", "");
- String regex = (post == null) ? ".*" : post.get("regex", ".*");
+ final String text = (post == null) ? "" : post.get("text", "");
+ final String regex = (post == null) ? ".*" : post.get("regex", ".*");
String error = "";
Boolean match = null;
try {
@@ -37,9 +39,9 @@ public class RegexTest {
} catch (final PatternSyntaxException e) {
error = e.getMessage();
}
-
+
final serverObjects prop = new serverObjects();
-
+
prop.put("text", text);
prop.put("regex", regex);
prop.put("match", match == null ? 2 : (match.booleanValue() ? 1 : 0));
diff --git a/htroot/RemoteCrawl_p.java b/source/net/yacy/htroot/RemoteCrawl_p.java
similarity index 92%
rename from htroot/RemoteCrawl_p.java
rename to source/net/yacy/htroot/RemoteCrawl_p.java
index 1117d47cf..102e71ad0 100644
--- a/htroot/RemoteCrawl_p.java
+++ b/source/net/yacy/htroot/RemoteCrawl_p.java
@@ -26,6 +26,8 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -53,7 +55,7 @@ public class RemoteCrawl_p {
sb.tables.recordAPICall(post, "RemoteCrawl_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "remote crawler configuration");
if (post.containsKey("crawlResponse")) {
- boolean crawlResponse = post.get("crawlResponse", "off").equals("on");
+ final boolean crawlResponse = post.get("crawlResponse", "off").equals("on");
// read remote crawl request settings
sb.initRemoteCrawler(crawlResponse);
@@ -71,11 +73,11 @@ public class RemoteCrawl_p {
// set seed information directly
sb.peers.mySeed().setFlagAcceptRemoteCrawl(sb.getConfigBool(SwitchboardConstants.CRAWLJOB_REMOTE, false));
-
+
// write remote crawl request settings
prop.put("disabled", !sb.peers.mySeed().isActive() && !sb.peers.mySeed().getFlagAcceptRemoteCrawl() ? 1 : 0);
prop.put("crawlResponse", sb.peers.mySeed().getFlagAcceptRemoteCrawl() ? 1 : 0);
- long RTCbusySleep = Math.max(1, env.getConfigLong(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, 100));
+ final long RTCbusySleep = Math.max(1, env.getConfigLong(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, 100));
final int RTCppm = (int) (60000L / RTCbusySleep);
prop.put("acceptCrawlLimit", RTCppm);
@@ -90,14 +92,14 @@ public class RemoteCrawl_p {
Iterator<Seed> e = null;
e = sb.peers.seedsSortedConnected(false, Seed.RCOUNT);
//e = sb.peers.seedsSortedConnected(false, yacySeed.LCOUNT);
- Pattern peerSearchPattern = null;
+ final Pattern peerSearchPattern = null;
while (e.hasNext() && conCount < 300) {
seed = e.next();
assert seed != null;
if (seed != null) {
final long lastseen = Math.abs((System.currentTimeMillis() - seed.getLastSeenUTC()) / 1000 / 60);
if (lastseen > 720) continue;
- long rcount = seed.getLong(Seed.RCOUNT, 0);
+ final long rcount = seed.getLong(Seed.RCOUNT, 0);
if (rcount == 0) continue;
if ((post != null && post.containsKey("search")) && peerSearchPattern != null /*(wrongregex == null)*/) {
boolean abort = true;
@@ -118,7 +120,7 @@ public class RemoteCrawl_p {
prop.putHTML(STR_TABLE_LIST + conCount + "_shortname", shortname);
prop.putHTML(STR_TABLE_LIST + conCount + "_peeraddress", peeradr);
prop.put(STR_TABLE_LIST + conCount + "_age", seed.getAge());
- String[] yv = yacyVersion.combined2prettyVersion(seed.get(Seed.VERSION, "0.1"), shortname);
+ final String[] yv = yacyVersion.combined2prettyVersion(seed.get(Seed.VERSION, "0.1"), shortname);
prop.putHTML(STR_TABLE_LIST + conCount + "_version", yv[0] + "/" + yv[1]);
prop.putNum(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen);
prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(Seed.UTC, "-"));
diff --git a/htroot/SearchAccessRate_p.java b/source/net/yacy/htroot/SearchAccessRate_p.java
similarity index 99%
rename from htroot/SearchAccessRate_p.java
rename to source/net/yacy/htroot/SearchAccessRate_p.java
index 62ad0853b..d3b60fc87 100644
--- a/htroot/SearchAccessRate_p.java
+++ b/source/net/yacy/htroot/SearchAccessRate_p.java
@@ -20,6 +20,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Properties;
import net.yacy.cora.protocol.RequestHeader;
diff --git a/htroot/SearchEventPicture.java b/source/net/yacy/htroot/SearchEventPicture.java
similarity index 96%
rename from htroot/SearchEventPicture.java
rename to source/net/yacy/htroot/SearchEventPicture.java
index 835506d01..cd58a7c24 100644
--- a/htroot/SearchEventPicture.java
+++ b/source/net/yacy/htroot/SearchEventPicture.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.peers.graphics.NetworkGraph;
import net.yacy.search.Switchboard;
diff --git a/htroot/ServerScannerList.java b/source/net/yacy/htroot/ServerScannerList.java
similarity index 97%
rename from htroot/ServerScannerList.java
rename to source/net/yacy/htroot/ServerScannerList.java
index e5dd62a26..df1defe36 100644
--- a/htroot/ServerScannerList.java
+++ b/source/net/yacy/htroot/ServerScannerList.java
@@ -18,6 +18,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.net.MalformedURLException;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
@@ -41,7 +43,7 @@ public class ServerScannerList {
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard)env;
- boolean edit = post != null && post.containsKey("edit");
+ final boolean edit = post != null && post.containsKey("edit");
prop.put("servertable_edit", edit ? 1 : 0);
prop.put("embedded", post == null ? 0 : post.containsKey("embedded") ? 1 : 0);
prop.put("servertable", 0);
diff --git a/htroot/SettingsAck_p.java b/source/net/yacy/htroot/SettingsAck_p.java
similarity index 95%
rename from htroot/SettingsAck_p.java
rename to source/net/yacy/htroot/SettingsAck_p.java
index 953e506ee..735372552 100644
--- a/htroot/SettingsAck_p.java
+++ b/source/net/yacy/htroot/SettingsAck_p.java
@@ -27,6 +27,8 @@
// javac -classpath .:../Classes SettingsAck_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.HashMap;
import java.util.Iterator;
import java.util.StringTokenizer;
@@ -66,7 +68,7 @@ public class SettingsAck_p {
prop.put("info", "1");//no information submitted
return prop;
}
-
+
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(header, post);
@@ -102,13 +104,13 @@ public class SettingsAck_p {
if (post.containsKey("proxyaccount")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=ProxyAccess");
-
+
/*
* display port info
*/
prop.put("info_port", env.getLocalPort());
prop.put("info_restart", "0");
-
+
// read and process data
String filter = (post.get("proxyfilter")).trim();
final boolean useProxyAccounts = post.containsKey("use_proxyaccounts") && post.get("use_proxyaccounts").equals("on");
@@ -156,21 +158,21 @@ public class SettingsAck_p {
// http networking
if (post.containsKey("httpNetworking")) {
-
+
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=ProxyAccess");
// set transparent proxy flag
- boolean isTransparentProxy = post.containsKey("isTransparentProxy");
+ final boolean isTransparentProxy = post.containsKey("isTransparentProxy");
env.setConfig(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, isTransparentProxy);
prop.put("info_isTransparentProxy", isTransparentProxy ? "on" : "off");
if (isTransparentProxy) prop.put("needsRestart", isTransparentProxy);
-
+
// set proxyAlwaysFresh flag
- boolean proxyAlwaysFresh = post.containsKey("proxyAlwaysFresh");
+ final boolean proxyAlwaysFresh = post.containsKey("proxyAlwaysFresh");
env.setConfig("proxyAlwaysFresh", proxyAlwaysFresh);
prop.put("info_proxyAlwaysFresh", proxyAlwaysFresh ? "on" : "off");
-
+
// setting via header property
env.setConfig("proxy.sendViaHeader", post.containsKey("proxy.sendViaHeader"));
prop.put("info_proxy.sendViaHeader", post.containsKey("proxy.sendViaHeader")? "on" : "off");
@@ -178,7 +180,7 @@ public class SettingsAck_p {
// setting X-Forwarded-for header property
env.setConfig("proxy.sendXForwardedForHeader", post.containsKey("proxy.sendXForwardedForHeader"));
prop.put("info_proxy.sendXForwardedForHeader", post.containsKey("proxy.sendXForwardedForHeader")? "on" : "off");
-
+
prop.put("info", "20");
return prop;
}
@@ -187,9 +189,9 @@ public class SettingsAck_p {
if (post.containsKey("serveraccount")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=ServerAccess");
-
+
// fileHost
- String fileHost = (post.get("fileHost")).trim();
+ final String fileHost = (post.get("fileHost")).trim();
if (fileHost != null && !fileHost.isEmpty() && !fileHost.equals(env.getConfig("fileHost", "localpeer"))) {
env.setConfig("fileHost", fileHost);
}
@@ -201,7 +203,7 @@ public class SettingsAck_p {
} else if (staticIP.startsWith("https://")) {
if (staticIP.length() > 8) { staticIP = staticIP.substring(8); } else { staticIP = ""; }
}
- String error = Seed.isProperIP(staticIP) ? null : "ip not proper: " + staticIP;
+ final String error = Seed.isProperIP(staticIP) ? null : "ip not proper: " + staticIP;
if (error == null) {
serverCore.useStaticIP = true;
sb.peers.mySeed().setIP(staticIP);
@@ -213,18 +215,18 @@ public class SettingsAck_p {
}
// publicPort
- String publicPort = (post.get("publicPort")).trim();
+ final String publicPort = (post.get("publicPort")).trim();
try {
- Integer pport = Integer.parseInt(publicPort);
+ final Integer pport = Integer.parseInt(publicPort);
if(pport < 65535 && pport >= 0) {
serverCore.usePublicPort = true;
sb.peers.mySeed().setPort(pport);
env.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, publicPort);
}
- } catch (NumberFormatException e) {
+ } catch (final NumberFormatException e) {
// noop
}
-
+
// server access data
String filter = (post.get("serverfilter")).trim();
/*String user = (String) post.get("serveruser");
@@ -274,19 +276,19 @@ public class SettingsAck_p {
prop.putHTML("info_filter", filter);
return prop;
}
-
+
/* Compression settings */
if (post.containsKey("serverCompression")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=ServerAccess");
-
+
final boolean oldValue = env.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT);
final boolean newValue = post.containsKey(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP);
env.setConfig(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP, newValue);
-
+
prop.put("needsRestart", newValue != oldValue);
-
+
/* server compression settings saved */
prop.put("info", "37");
return prop;
@@ -334,7 +336,7 @@ public class SettingsAck_p {
if (post.containsKey("urlproxySettings")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=UrlProxyAccess");
-
+
env.setConfig("proxyURL.access", post.get("urlproxyfilter"));
env.setConfig("proxyURL.rewriteURLs", post.get("urlproxydomains"));
env.setConfig("proxyURL", "on".equals(post.get("urlproxyenabled")) ? true : false);
@@ -347,7 +349,7 @@ public class SettingsAck_p {
if (post.containsKey("seedUploadRetry")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=seed");
-
+
String error;
if ((error = Network.saveSeedList(sb)) == null) {
// trying to upload the seed-list file
@@ -364,7 +366,7 @@ public class SettingsAck_p {
if (post.containsKey("seedSettings")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=seed");
-
+
// get the currently used uploading method
final String oldSeedUploadMethod = env.getConfig("seedUploadMethod","none");
final String newSeedUploadMethod = post.get("seedUploadMethod");
@@ -381,7 +383,7 @@ public class SettingsAck_p {
env.setConfig("seedUploadMethod", newSeedUploadMethod);
sb.peers.mySeed().put(Seed.SEEDLISTURL, newSeedURLStr);
if (seedUrlChanged) sb.peers.saveMySeed();
-
+
// try an upload
String error;
if ((error = Network.saveSeedList(sb)) == null) {
@@ -398,7 +400,7 @@ public class SettingsAck_p {
return prop;
}
}
-
+
if (post.containsKey("seedFileSettings") || post.containsKey("seedFtpSettings") || post.containsKey("seedScpSettings")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=seed");
@@ -465,7 +467,7 @@ public class SettingsAck_p {
if (post.containsKey("msgForwarding")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=messageForwarding");
-
+
env.setConfig("msgForwardingEnabled", post.containsKey("msgForwardingEnabled"));
env.setConfig("msgForwardingCmd", post.get("msgForwardingCmd"));
env.setConfig("msgForwardingTo", post.get("msgForwardingTo"));
@@ -480,7 +482,7 @@ public class SettingsAck_p {
// Crawler settings
if (post.containsKey("crawlerSettings")) {
-
+
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=crawler");
@@ -568,11 +570,11 @@ public class SettingsAck_p {
// server port settings
if (post.containsKey("serverports")) {
- int port = post.getInt("port", 8090);
+ final int port = post.getInt("port", 8090);
if (port > 0) env.setConfig(SwitchboardConstants.SERVER_PORT, port);
- int portssl = post.getInt("port.ssl", 8443);
+ final int portssl = post.getInt("port.ssl", 8443);
if (portssl > 0) env.setConfig(SwitchboardConstants.SERVER_SSLPORT, portssl);
- int portshutdown = post.getInt("port.shutdown", -1);
+ final int portshutdown = post.getInt("port.shutdown", -1);
env.setConfig(SwitchboardConstants.SERVER_SHUTDOWNPORT, portshutdown);
prop.put("info_port", port);
prop.put("info_port.ssl", portssl);
@@ -584,11 +586,11 @@ public class SettingsAck_p {
// change https port
if (post.containsKey("port.ssl")) {
-
+
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=ProxyAccess");
-
- int port = post.getInt("port.ssl", 8443);
+
+ final int port = post.getInt("port.ssl", 8443);
if (port > 0 && port != env.getConfigInt(SwitchboardConstants.SERVER_SSLPORT, 8443)) {
env.setConfig(SwitchboardConstants.SERVER_SSLPORT, port);
}
@@ -596,80 +598,80 @@ public class SettingsAck_p {
prop.put("info", "32");
return prop;
}
-
+
// Debug/Analysis settings
if (post.containsKey("debugAnalysisSettings")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=debug");
-
+
boolean tickedCheckbox = post.containsKey("solrBinaryResponse");
env.setConfig(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED, tickedCheckbox);
-
+
tickedCheckbox = post.containsKey("searchTestLocalDHT");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, tickedCheckbox);
tickedCheckbox = post.containsKey("searchTestLocalSolr");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, tickedCheckbox);
-
+
tickedCheckbox = post.containsKey("searchShowRanking");
env.setConfig(SwitchboardConstants.SEARCH_RESULT_SHOW_RANKING, tickedCheckbox);
-
+
tickedCheckbox = post.containsKey(SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED);
sb.setConfig(SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED, tickedCheckbox);
TextSnippet.statistics.setEnabled(tickedCheckbox);
-
- /* For easier user understanding, the following flags controlling data sources selection
+
+ /* For easier user understanding, the following flags controlling data sources selection
* are rendered in the UI as checkboxes corresponding to enabled value when ticked */
tickedCheckbox = post.containsKey("searchLocalDHT");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_OFF, !tickedCheckbox);
-
+
tickedCheckbox = post.containsKey("searchLocalSolr");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, !tickedCheckbox);
-
+
tickedCheckbox = post.containsKey("searchRemoteDHT");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, !tickedCheckbox);
tickedCheckbox = post.containsKey("searchRemoteSolr");
env.setConfig(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, !tickedCheckbox);
-
+
/* Let's clean up all search events as these settings affect how search is performed and we don't
* want cached results obtained with the previous settings */
SearchEventCache.cleanupEvents(true);
-
+
prop.put("info", "34");
return prop;
}
-
+
// Referrer Policy settings
if (post.containsKey("referrerPolicySettings")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=referrer");
-
- String metaPolicy = post.get("metaPolicy", SwitchboardConstants.REFERRER_META_POLICY_DEFAULT);
+
+ final String metaPolicy = post.get("metaPolicy", SwitchboardConstants.REFERRER_META_POLICY_DEFAULT);
env.setConfig(SwitchboardConstants.REFERRER_META_POLICY, metaPolicy);
-
- boolean tickedCheckbox = post.containsKey("searchResultNoReferrer");
+
+ final boolean tickedCheckbox = post.containsKey("searchResultNoReferrer");
env.setConfig(SwitchboardConstants.SEARCH_RESULT_NOREFERRER, tickedCheckbox);
-
+
prop.put("info", "35");
return prop;
}
-
+
// HTTP client settings
if (post.containsKey("httpClientSettings")) {
// set backlink
prop.put("needsRestart_referer", "Settings_p.html?page=httpClient");
-
+
if(System.getProperty("jsse.enableSNIExtension") == null) {
/* Only apply custom SNI extension settings when the JVM system option jsse.enableSNIExtension is not defined */
env.setConfig(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED,
post.containsKey(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED));
-
+
env.setConfig(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED,
post.containsKey(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED));
}
sb.initOutgoingConnectionSettings();
-
+
prop.put("info", "38");
return prop;
}
diff --git a/htroot/Settings_p.java b/source/net/yacy/htroot/Settings_p.java
similarity index 95%
rename from htroot/Settings_p.java
rename to source/net/yacy/htroot/Settings_p.java
index 329843394..fc08514e6 100644
--- a/htroot/Settings_p.java
+++ b/source/net/yacy/htroot/Settings_p.java
@@ -22,6 +22,8 @@
// javac -classpath .:../Classes Settings_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.HashMap;
import java.util.Iterator;
@@ -40,19 +42,19 @@ import net.yacy.server.serverSwitch;
public final class Settings_p {
-
+
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
-
+
//if (post == null) System.out.println("POST: NULL"); else System.out.println("POST: " + post.toString());
-
+
final String page = (post == null) ? "general" : post.get("page", "general");
-
+
/* Acquire a transaction token for the next possible POST form submission */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header, "/SettingsAck_p.html"));
-
+
if (page.equals("ProxyAccess")) {
prop.put("settingsTables", "Settings_ProxyAccess.inc");
}
@@ -62,9 +64,9 @@ public final class Settings_p {
else if (page.equals("UrlProxyAccess")) {
prop.put("settingsTables", "Settings_UrlProxyAccess.inc");
}
- else if (page.equals("ServerAccess")) {
+ else if (page.equals("ServerAccess")) {
prop.put("settingsTables", "Settings_ServerAccess.inc");
- } else if (page.equals("referrer")) {
+ } else if (page.equals("referrer")) {
prop.put("settingsTables", "Settings_Referrer.inc");
}
else if (page.equals("SystemBehaviour")) {
@@ -90,7 +92,7 @@ public final class Settings_p {
}
prop.put("port", env.getLocalPort());
-
+
prop.putHTML("peerName", sb.peers.mySeed().getName());
prop.putHTML("staticIP", env.getConfig("staticIP", ""));
prop.putHTML("publicPort", env.getConfig("publicPort",""));
@@ -98,28 +100,28 @@ public final class Settings_p {
String peerLang = env.getConfig("locale.language", "default");
if (peerLang.equals("default")) peerLang = "en";
prop.putHTML("peerLang", peerLang);
-
+
// http networking settings
prop.put("isTransparentProxy", env.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "1" : "0");
prop.put("proxyAlwaysFresh", env.getConfigBool("proxyAlwaysFresh", false) ? "1" : "0");
prop.put("proxy.sendViaHeader", env.getConfigBool("proxy.sendViaHeader", false) ? "1" : "0");
prop.put("proxy.sendXForwardedForHeader", env.getConfigBool("proxy.sendXForwardedForHeader", true) ? "1" : "0");
-
+
// remote proxy
prop.put("remoteProxyUseChecked", env.getConfigBool("remoteProxyUse", false) ? 1 : 0);
prop.put("remoteProxyUse4SSL", env.getConfigBool("remoteProxyUse4SSL", true) ? 1 : 0);
-
+
prop.putHTML("remoteProxyHost", env.getConfig("remoteProxyHost", ""));
prop.putHTML("remoteProxyPort", env.getConfig("remoteProxyPort", ""));
-
+
prop.putHTML("remoteProxyUser", env.getConfig("remoteProxyUser", ""));
prop.putHTML("remoteProxyPwd", env.getConfig("remoteProxyPwd", ""));
-
+
prop.putHTML("remoteProxyNoProxy", env.getConfig("remoteProxyNoProxy", ""));
-
+
// proxy access filter
prop.putHTML("proxyfilter", env.getConfig("proxyClient", "*"));
-
+
// proxy password
if (!env.getConfigBool("use_proxyAccounts", false)) {
// no password has been specified
@@ -143,47 +145,47 @@ public final class Settings_p {
// server access filter
prop.putHTML("serverfilter", env.getConfig("serverClient", "*"));
-
+
/* gzip compression of HTTP responses */
prop.put(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
env.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT));
-
+
// server password
prop.put("serveruser","server");
-
+
// clientIP
prop.putXML("clientIP", header.getRemoteAddr() == null ? "<unknown>" : header.getRemoteAddr());
- /*
+ /*
* seed upload settings
*/
// available methods
String enabledUploader = env.getConfig("seedUploadMethod", "none");
-
+
// for backward compatiblity ....
- if ((enabledUploader.equalsIgnoreCase("Ftp")) ||
+ if ((enabledUploader.equalsIgnoreCase("Ftp")) ||
((enabledUploader.equals("")) &&
(env.getConfig("seedFTPPassword","").length() > 0) &&
(env.getConfig("seedFilePath", "").length() > 0))) {
enabledUploader = "Ftp";
env.setConfig("seedUploadMethod",enabledUploader);
- }
-
+ }
+
final HashMap<String, String> uploaders = Network.getSeedUploadMethods();
prop.put("seedUploadMethods", uploaders.size() + 1);
prop.put("seedUploadMethods_0_name", "none");
prop.put("seedUploadMethods_0_selected", enabledUploader.equals("none") ? "1" : "0");
prop.put("seedUploadMethods_0_file", "");
-
+
int count = 0;
final Iterator<String> uploaderKeys = uploaders.keySet().iterator();
while (uploaderKeys.hasNext()) {
count++;
final String uploaderName = uploaderKeys.next();
prop.put("seedUploadMethods_" +count+ "_name", uploaderName);
- prop.put("seedUploadMethods_" +count+ "_selected", uploaderName.equals(enabledUploader) ? "1" : "0");
+ prop.put("seedUploadMethods_" +count+ "_selected", uploaderName.equals(enabledUploader) ? "1" : "0");
prop.put("seedUploadMethods_" +count+ "_file", "Settings_Seed_Upload" + uploaderName + ".inc");
-
+
final yacySeedUploader theUploader = Network.getSeedUploader(uploaderName);
final String[] configOptions = theUploader.getConfigurationOptions();
if (configOptions != null) {
@@ -193,10 +195,10 @@ public final class Settings_p {
}
}
}
-
+
// general settings
prop.put("seedURL", sb.peers.mySeed().get(Seed.SEEDLISTURL, ""));
-
+
/*
* Message forwarding configuration
*/
@@ -215,9 +217,9 @@ public final class Settings_p {
prop.put("server.https", sb.getConfigBool("server.https", false));
prop.put("server.https_port.ssl", sb.getConfig(SwitchboardConstants.SERVER_SSLPORT,"8443"));
prop.put("port.shutdown", sb.getConfig(SwitchboardConstants.SERVER_SHUTDOWNPORT, "-1"));
-
+
// Referrer Policy
- final String metaPolicy = env.getConfig(SwitchboardConstants.REFERRER_META_POLICY,
+ final String metaPolicy = env.getConfig(SwitchboardConstants.REFERRER_META_POLICY,
SwitchboardConstants.REFERRER_META_POLICY_DEFAULT);
prop.put("metaPolicyNoReferrerChecked", ReferrerPolicy.NO_REFERRER.getValue().equals(metaPolicy) ? 1 : 0);
prop.put("metaPolicySameOriginChecked", ReferrerPolicy.SAME_ORIGIN.getValue().equals(metaPolicy) ? 1 : 0);
@@ -235,14 +237,14 @@ public final class Settings_p {
prop.put("metaPolicyCustom_checked", 1);
prop.put("metaPolicyCustom_value", metaPolicy);
}
-
- prop.put("searchResultNoReferrerChecked", env.getConfigBool(SwitchboardConstants.SEARCH_RESULT_NOREFERRER,
+
+ prop.put("searchResultNoReferrerChecked", env.getConfigBool(SwitchboardConstants.SEARCH_RESULT_NOREFERRER,
SwitchboardConstants.SEARCH_RESULT_NOREFERRER_DEFAULT) ? 1 : 0);
-
+
// debug/analysis
- prop.put("solrBinaryResponseChecked", env.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,
+ prop.put("solrBinaryResponseChecked", env.getConfigBool(SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED,
SwitchboardConstants.REMOTE_SOLR_BINARY_RESPONSE_ENABLED_DEFAULT) ? 1 : 0);
-
+
// HTTP client
prop.put(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED,
env.getConfigBool(SwitchboardConstants.HTTP_OUTGOING_GENERAL_TLS_SNI_EXTENSION_ENABLED,
@@ -259,25 +261,25 @@ public final class Settings_p {
jvmSniExtensionSetting != null);
prop.put(SwitchboardConstants.HTTP_OUTGOING_REMOTE_SOLR_TLS_SNI_EXTENSION_ENABLED + ".readonly_jvmSettingValue",
jvmSniExtensionSetting);
-
- /* For easier user understanding, the following flags controlling data sources selection
+
+ /* For easier user understanding, the following flags controlling data sources selection
* are rendered in the UI as checkboxes corresponding to enabled value when ticked */
prop.put("searchLocalDHTChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_DHT_OFF, false) ? 1 : 0);
prop.put("searchLocalSolrChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false) ? 1 : 0);
prop.put("searchRemoteDHTChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_OFF, false) ? 1 : 0);
prop.put("searchRemoteSolrChecked", !env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_OFF, false) ? 1 : 0);
-
+
prop.put("searchTestLocalDHTChecked", env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_DHT_TESTLOCAL, false) ? 1 : 0);
prop.put("searchTestLocalSolrChecked", env.getConfigBool(SwitchboardConstants.DEBUG_SEARCH_REMOTE_SOLR_TESTLOCAL, false) ? 1 : 0);
-
+
prop.put("searchShowRankingChecked", env.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_RANKING, SwitchboardConstants.SEARCH_RESULT_SHOW_RANKING_DEFAULT) ? 1 : 0);
-
+
prop.put(SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED,
sb.getConfigBool(SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED,
SwitchboardConstants.DEBUG_SNIPPETS_STATISTICS_ENABLED_DEFAULT));
-
+
// return rewrite properties
return prop;
}
-
+
}
diff --git a/htroot/Status.java b/source/net/yacy/htroot/Status.java
similarity index 96%
rename from htroot/Status.java
rename to source/net/yacy/htroot/Status.java
index 0899bb341..dede1e48e 100644
--- a/htroot/Status.java
+++ b/source/net/yacy/htroot/Status.java
@@ -27,6 +27,8 @@
// javac -classpath .:../Classes Status.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.util.Date;
@@ -142,7 +144,7 @@ public class Status
prop.put("unrestrictedLocalAccess", 1);
if(sb.getConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, "").indexOf("ConfigAccounts_p.html", 0) < 0) {
/* Encourage checking accounts config page to be sure that unrestricted local access is desired */
- prop.put("warningUnrestrictedLocalAccess", true);
+ prop.put("warningUnrestrictedLocalAccess", true);
}
}
@@ -196,13 +198,13 @@ public class Status
prop.put("extPortFormat", "0");
}
- Set<String> ips = Domains.myPublicIPs();
+ final Set<String> ips = Domains.myPublicIPs();
prop.put("host", ips.toString());
// ssl support
prop.put("sslSupport", sb.getConfig("keyStore", "").isEmpty() || !sb.getConfigBool("server.https", false) ? 0 : 1);
if (sb.getConfigBool("server.https", false)) prop.put("sslSupport_sslPort", sb.getHttpServer().getSslPort());
-
+
// proxy information
if ( sb.getConfigBool("remoteProxyUse", false) ) {
prop.put("remoteProxy", "1");
@@ -213,7 +215,7 @@ public class Status
}
prop.put("info_isTransparentProxy", sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false) ? "0" : "1");
prop.put("info_proxyURL", sb.getConfigBool("proxyURL", false) ? "0" : "1");
-
+
// peer information
String thisHash = "";
final String thisName = sb.peers.mySeed().getName();
@@ -321,7 +323,7 @@ public class Status
prop.put("freeDisk", Formatter.bytesToString(sb.observer.getUsableSpace()));
prop.put("processors", WorkflowProcessor.availableCPU);
prop.put("load", Memory.getSystemLoadAverage());
- ThreadMXBean threadbean = ManagementFactory.getThreadMXBean();
+ final ThreadMXBean threadbean = ManagementFactory.getThreadMXBean();
prop.put("processesCurrentInclDaemon", threadbean.getThreadCount());
prop.put("processesCurrentOnlyDaemon", threadbean.getDaemonThreadCount());
prop.put("processesTotal", threadbean.getTotalStartedThreadCount());
diff --git a/htroot/Steering.java b/source/net/yacy/htroot/Steering.java
similarity index 96%
rename from htroot/Steering.java
rename to source/net/yacy/htroot/Steering.java
index e83cb5460..f59116710 100644
--- a/htroot/Steering.java
+++ b/source/net/yacy/htroot/Steering.java
@@ -27,6 +27,8 @@
// javac -classpath .:../Classes SettingsAck_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import net.yacy.cora.protocol.Domains;
@@ -45,10 +47,10 @@ public class Steering {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch ss) {
if (post == null || post.isEmpty() || ss == null) {
final serverObjects prop = new serverObjects();
-
+
/* For authenticated user only : acquire a transaction token to pass then to the Steering.html post action */
if(ss != null && ((Switchboard) ss).verifyAuthentication(header)) {
- /* YaCyDefaultServlet will detect it and then also fill the custom HTTP response header used by the JavaScript shutdown and restart actions
+ /* YaCyDefaultServlet will detect it and then also fill the custom HTTP response header used by the JavaScript shutdown and restart actions
* or any external API requesting tool */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));
/* Also add to the Steering.html page info block for eventual display of this page without parameter */
@@ -56,7 +58,7 @@ public class Steering {
} else {
prop.authenticationRequired();
}
-
+
return prop;
}
@@ -115,7 +117,7 @@ public class Steering {
return prop;
}
-
+
return prop;
}
diff --git a/htroot/Supporter.java b/source/net/yacy/htroot/Supporter.java
similarity index 99%
rename from htroot/Supporter.java
rename to source/net/yacy/htroot/Supporter.java
index 02f44197b..ca05c1e01 100644
--- a/htroot/Supporter.java
+++ b/source/net/yacy/htroot/Supporter.java
@@ -25,6 +25,8 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.net.MalformedURLException;
import java.util.Date;
import java.util.HashMap;
diff --git a/htroot/Surftips.java b/source/net/yacy/htroot/Surftips.java
similarity index 99%
rename from htroot/Surftips.java
rename to source/net/yacy/htroot/Surftips.java
index 893a27a5d..47e220b38 100644
--- a/htroot/Surftips.java
+++ b/source/net/yacy/htroot/Surftips.java
@@ -25,6 +25,8 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.net.MalformedURLException;
import java.util.Date;
import java.util.HashMap;
diff --git a/htroot/Table_API_p.java b/source/net/yacy/htroot/Table_API_p.java
similarity index 95%
rename from htroot/Table_API_p.java
rename to source/net/yacy/htroot/Table_API_p.java
index 24400f3a7..f8c7d1a13 100644
--- a/htroot/Table_API_p.java
+++ b/source/net/yacy/htroot/Table_API_p.java
@@ -17,6 +17,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.DateFormat;
@@ -59,9 +61,9 @@ public class Table_API_p {
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
-
+
final DateFormat dateFormat = GenericFormatter.newSimpleDateFormat();
-
+
prop.put("showexec", 0);
prop.put("showtable", 0);
@@ -87,7 +89,7 @@ public class Table_API_p {
if (post != null && post.containsKey("filter") && post.get("filter", "").length() > 0) {
typefilter = Pattern.compile(post.get("filter", ".*"));
}
-
+
/* Applying JSON API convention for the sort parameter format (http://jsonapi.org/format/#fetching-sorting) */
String sortParam = WorkTables.TABLE_API_COL_DATE_RECORDING;
if(post != null) {
@@ -112,38 +114,38 @@ public class Table_API_p {
current_pk = post.get("current_pk", "");
}
if (post != null && scheduleeventaction && !current_pk.isEmpty()) {
-
+
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(header, post);
-
+
try {
- Tables.Row row = sb.tables.select(WorkTables.TABLE_API_NAME, current_pk.getBytes());
+ final Tables.Row row = sb.tables.select(WorkTables.TABLE_API_NAME, current_pk.getBytes());
if (row != null) {
String action;
-
+
// events
if (post.containsKey("event_select_" + current_pk) && post.get("event_select_" + current_pk, "off").equals("on")) {
row.put(WorkTables.TABLE_API_COL_APICALL_EVENT_KIND, "regular");
row.put(WorkTables.TABLE_API_COL_APICALL_EVENT_ACTION, "startup");
}
-
+
if (post.containsKey("event_kind_" + current_pk) ) {
if ("off".equals(action = post.get("event_kind_" + current_pk, "off"))) {
row.put(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, "");
}
row.put(WorkTables.TABLE_API_COL_APICALL_EVENT_KIND, action);
}
-
+
if (post.containsKey("event_action_" + current_pk) ) {
row.put(WorkTables.TABLE_API_COL_APICALL_EVENT_ACTION, post.get("event_action_" + current_pk, "startup"));
}
-
+
// scheduler
if (post.containsKey("repeat_select_" + current_pk) && post.get("repeat_select_" + current_pk, "off").equals("on")) {
row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 7);
row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_UNIT, "days");
}
-
+
if (post.containsKey("repeat_time_" + current_pk) ) {
if ("off".equals(action = post.get("repeat_time_" + current_pk, "off"))) {
row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 0);
@@ -151,10 +153,10 @@ public class Table_API_p {
row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, Integer.parseInt(action));
}
}
-
+
if (post.containsKey("repeat_unit_" + current_pk) ) {
action = post.get("repeat_unit_" + current_pk, "seldays");
- int time = row.get(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 1);
+ final int time = row.get(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 1);
row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_UNIT, action.substring(3));
if (action.equals("selminutes") && time > 0 && time < 10) {
row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 10);
@@ -169,26 +171,26 @@ public class Table_API_p {
row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 30);
}
}
-
+
// switch scheduler off if event kind is 'regular'
final String kind = row.get(WorkTables.TABLE_API_COL_APICALL_EVENT_KIND, "off");
if ("regular".equals(kind)) row.put(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, 0);
-
+
WorkTables.calculateAPIScheduler(row, false);
sb.tables.update(WorkTables.TABLE_API_NAME, row);
}
} catch (final Throwable e) { ConcurrentLog.logException(e); }
}
-
+
/* Edition of scheduled next execution dates */
final Map<String, String> invalidNextExecDateFormats = new HashMap<>();
final Map<String, String> nextExecDatesBeforeNow = new HashMap<>();
if (post != null && post.containsKey("submitNextExecDates")) {
-
+
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(header, post);
-
-
+
+
final String dateNexExecPrefix = "date_next_exec_";
final Date now = new Date();
for (final Map.Entry<String, String> entry : post.entrySet()) {
@@ -202,20 +204,20 @@ public class Table_API_p {
final String dateNextExecStr = entry.getValue().trim();
try {
final Date dateNextExec = dateFormat.parse(dateNextExecStr);
-
- if(time != 0) { // Check there is effectively a schedule period on this row
+
+ if(time != 0) { // Check there is effectively a schedule period on this row
if(dateNextExec.before(now)) {
nextExecDatesBeforeNow.put(rowPkStr, dateNextExecStr);
} else {
row.put(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, dateNextExec);
-
+
sb.tables.update(WorkTables.TABLE_API_NAME, row);
}
}
} catch (final ParseException e) {
invalidNextExecDateFormats.put(rowPkStr, dateNextExecStr);
}
-
+
}
} catch (final IOException | SpaceExceededException e) {
ConcurrentLog.logException(e);
@@ -223,12 +225,12 @@ public class Table_API_p {
}
}
}
-
+
if (post != null && !post.get("deleterows", "").isEmpty()) {
-
+
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(header, post);
-
+
for (final Map.Entry<String, String> entry : post.entrySet()) {
if (entry.getValue().startsWith("mark_")) {
try {
@@ -239,32 +241,32 @@ public class Table_API_p {
}
}
}
-
+
if (post != null && !post.get("deleteold", "").isEmpty()) {
-
+
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(header, post);
-
- int days = post.getInt("deleteoldtime", 365);
+
+ final int days = post.getInt("deleteoldtime", 365);
try {
- Iterator<Row> ri = sb.tables.iterator(WorkTables.TABLE_API_NAME);
+ final Iterator<Row> ri = sb.tables.iterator(WorkTables.TABLE_API_NAME);
Row row;
- Date now = new Date();
- Date limit = new Date(now.getTime() - AbstractFormatter.dayMillis * days);
- List<byte[]> pkl = new ArrayList<byte[]>();
+ final Date now = new Date();
+ final Date limit = new Date(now.getTime() - AbstractFormatter.dayMillis * days);
+ final List<byte[]> pkl = new ArrayList<byte[]>();
while (ri.hasNext()) {
row = ri.next();
- Date d = row.get(WorkTables.TABLE_API_COL_DATE_RECORDING, now);
+ final Date d = row.get(WorkTables.TABLE_API_COL_DATE_RECORDING, now);
if (d.before(limit)) {
pkl.add(row.getPK());
}
}
- for (byte[] pk: pkl) {
+ for (final byte[] pk: pkl) {
sb.tables.delete(WorkTables.TABLE_API_NAME, pk);
}
-
+
// store this call as api call; clean the call a bit before
- Iterator<Entry<String, String[]>> ei = post.getSolrParams().getMap().entrySet().iterator();
+ final Iterator<Entry<String, String[]>> ei = post.getSolrParams().getMap().entrySet().iterator();
Entry<String, String[]> entry;
while (ei.hasNext()) {
entry = ei.next();
@@ -282,10 +284,10 @@ public class Table_API_p {
}
if (post != null && !post.get("execrows", "").isEmpty()) {
-
+
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(header, post);
-
+
// create a time-ordered list of events to execute
final Set<String> pks = new TreeSet<String>();
for (final Map.Entry<String, String> entry : post.entrySet()) {
@@ -317,16 +319,16 @@ public class Table_API_p {
}
prop.put("showexec_list", count);
}
-
+
// generate table
prop.put("showtable", 1);
prop.put("showtable_inline", inline ? 1 : 0);
-
+
/* Acquire a transaction token for the next POST form submission */
final String nextTransactionToken = TransactionManager.getTransactionToken(header);
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken);
prop.put("showtable_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken);
-
+
final Date now = new Date();
// insert rows
@@ -361,7 +363,7 @@ public class Table_API_p {
String type, comment, url;
final boolean hasFilter = typefilter != QueryParams.catchall_pattern
|| query != QueryParams.catchall_pattern;
-
+
// first prepare a list
while (mapIterator.hasNext()) {
r = mapIterator.next();
@@ -409,7 +411,7 @@ public class Table_API_p {
} else {
filteredSize = tablesize;
}
-
+
// then work on the list
for (final Tables.Row row : table) {
final String rowPKStr = UTF8.String(row.getPK());
@@ -427,24 +429,24 @@ public class Table_API_p {
prop.put("showtable_list_" + count + "_callcount", callcount);
prop.put("showtable_list_" + count + "_dateRecording", date_recording == null ? "-" : dateFormat.format(date_recording));
prop.put("showtable_list_" + count + "_dateLastExec", date_last_exec == null ? "-" : dateFormat.format(date_last_exec));
-
+
prop.put("showtable_list_" + count + "_editableDateNext", time != 0);
final String enteredDateBeforeNow = nextExecDatesBeforeNow.get(rowPKStr);
prop.put("showtable_list_" + count + "_editableDateNext_dateBeforeNowError", enteredDateBeforeNow != null);
if(enteredDateBeforeNow != null) {
prop.put("showtable_list_" + count + "_editableDateNext_dateBeforeNowError_invalidDate", enteredDateBeforeNow);
}
-
+
final String invalidEnteredDate = invalidNextExecDateFormats.get(rowPKStr);
prop.put("showtable_list_" + count + "_editableDateNext_dateFormatError", invalidEnteredDate != null);
if(invalidEnteredDate != null) {
prop.put("showtable_list_" + count + "_editableDateNext_dateFormatError_invalidDate", invalidEnteredDate);
}
-
+
prop.put("showtable_list_" + count + "_editableDateNext_dateLastExecPattern", GenericFormatter.PATTERN_SIMPLE_REGEX);
prop.put("showtable_list_" + count + "_editableDateNext_dateNextExec", date_next_exec == null ? "-" : dateFormat.format(date_next_exec));
prop.put("showtable_list_" + count + "_editableDateNext_pk", rowPKStr);
-
+
prop.put("showtable_list_" + count + "_type", row.get(WorkTables.TABLE_API_COL_TYPE));
prop.putHTML("showtable_list_" + count + "_comment", row.get(WorkTables.TABLE_API_COL_COMMENT));
// check type & action to link crawl start URLs back to CrawlStartExpert.html
@@ -452,13 +454,13 @@ public class Table_API_p {
&& prop.get("showtable_list_" + count + "_comment", "").startsWith("crawl start for")) {
final String editUrl = UTF8.String(row.get(WorkTables.TABLE_API_COL_URL)).replace("Crawler_p", "CrawlStartExpert");
if (editUrl.length() > 1000) {
- final MultiProtocolURL u = new MultiProtocolURL("http://localhost:8090" + editUrl);
+ final MultiProtocolURL u = new MultiProtocolURL("http://localhost:8090" + editUrl);
prop.put("showtable_list_" + count + "_isCrawlerStart", 2);
prop.put("showtable_list_" + count + "_isCrawlerStart_pk", rowPKStr);
prop.put("showtable_list_" + count + "_isCrawlerStart_servlet", "CrawlStartExpert.html");
- Map<String, String> attr = u.getAttributes();
+ final Map<String, String> attr = u.getAttributes();
int ac = 0;
- for (Map.Entry<String, String> entry: attr.entrySet()) {
+ for (final Map.Entry<String, String> entry: attr.entrySet()) {
prop.put("showtable_list_" + count + "_isCrawlerStart_attr_" + ac + "_key", entry.getKey());
prop.put("showtable_list_" + count + "_isCrawlerStart_attr_" + ac + "_value", entry.getValue());
ac++;
@@ -467,7 +469,7 @@ public class Table_API_p {
} else {
// short calls
prop.put("showtable_list_" + count + "_isCrawlerStart", 1);
- /* For better integration of YaCy peers behind a reverse proxy subfolder,
+ /* For better integration of YaCy peers behind a reverse proxy subfolder,
* ensure a path relative to this servlet (with no starting slash) is used for clone links.
* We keep the paths starting with a slash for other URL displays. */
prop.put("showtable_list_" + count + "_isCrawlerStart_url", editUrl.startsWith("/") ? editUrl.substring(1, editUrl.length()) : editUrl);
@@ -481,12 +483,12 @@ public class Table_API_p {
prop.put("showtable_list_" + count + "_scheduler_query", query.pattern());
prop.put("showtable_list_" + count + "_scheduler_startRecord", startRecord);
prop.put("showtable_list_" + count + "_scheduler_maximumRecords", maximumRecords);
-
+
// events
final String kind = row.get(WorkTables.TABLE_API_COL_APICALL_EVENT_KIND, "off");
final String action = row.get(WorkTables.TABLE_API_COL_APICALL_EVENT_ACTION, "startup");
prop.put("showtable_list_" + count + "_event_pk", rowPKStr);
- boolean schedulerDisabled = "regular".equals(kind);
+ final boolean schedulerDisabled = "regular".equals(kind);
if ("off".equals(kind)) {
prop.put("showtable_list_" + count + "_event", 0);
} else {
@@ -545,7 +547,7 @@ public class Table_API_p {
prop.put("showtable_list_" + count + "_scheduler_scale", 31);
prop.put("showtable_list_" + count + "_scheduler_selectedDays", 1);
}
-
+
}
count++;
}
@@ -569,9 +571,9 @@ public class Table_API_p {
prop.put("showtable_filter", typefilter.pattern());
prop.put("showtable_query", queryParam);
prop.put("showtable_sort", sortParam);
-
+
putTableSortProperties(prop, sortDir, sortColumn);
-
+
if (filteredSize > maximumRecords) {
prop.put("showtable_navigation", 1);
prop.put("showtable_navigation_startRecord", startRecord);
@@ -603,7 +605,7 @@ public class Table_API_p {
/**
* Fill the serverObjects instance with table columns sort properties.
- *
+ *
* @param prop
* the serverObjects instance to fill. Must not be null.
* @param sortDir
diff --git a/htroot/Table_RobotsTxt_p.java b/source/net/yacy/htroot/Table_RobotsTxt_p.java
similarity index 91%
rename from htroot/Table_RobotsTxt_p.java
rename to source/net/yacy/htroot/Table_RobotsTxt_p.java
index edbfd9ebf..6be9ac448 100644
--- a/htroot/Table_RobotsTxt_p.java
+++ b/source/net/yacy/htroot/Table_RobotsTxt_p.java
@@ -1,3 +1,5 @@
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
diff --git a/htroot/Tables_p.java b/source/net/yacy/htroot/Tables_p.java
similarity index 96%
rename from htroot/Tables_p.java
rename to source/net/yacy/htroot/Tables_p.java
index eaf2f5af1..c9a11016f 100644
--- a/htroot/Tables_p.java
+++ b/source/net/yacy/htroot/Tables_p.java
@@ -17,6 +17,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -44,7 +46,7 @@ public class Tables_p {
final String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, "");
//final boolean json = ext.equals("json");
final boolean xml = ext.equals("xml");
-
+
prop.put("showtable", 0);
prop.put("showedit", 0);
prop.put("showselection", 0);
@@ -172,7 +174,7 @@ public class Tables_p {
prop.putXML("showtable_" + count + "_table", table); // only used in XML
byte[] cell;
while (plainIterator.hasNext() && count < maxcount) {
- Tables.Row row = plainIterator.next();
+ final Tables.Row row = plainIterator.next();
if (row == null) continue;
// write table content
diff --git a/htroot/Threaddump_p.java b/source/net/yacy/htroot/Threaddump_p.java
similarity index 86%
rename from htroot/Threaddump_p.java
rename to source/net/yacy/htroot/Threaddump_p.java
index de52b3fd9..58b5b4390 100644
--- a/htroot/Threaddump_p.java
+++ b/source/net/yacy/htroot/Threaddump_p.java
@@ -25,6 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.logging.ThreadDump;
import net.yacy.search.Switchboard;
@@ -35,8 +37,8 @@ public class Threaddump_p {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
- serverObjects prop = new serverObjects();
- Switchboard sb = (Switchboard) env;
+ final serverObjects prop = new serverObjects();
+ final Switchboard sb = (Switchboard) env;
final boolean plain = post != null && post.getBoolean("plain");
final int sleep = (post == null) ? 0 : post.getInt("sleep", 0); // a sleep before creation of a thread dump can be used for profiling
@@ -44,12 +46,12 @@ public class Threaddump_p {
prop.put("dump", "1");
int multipleCount = 100;
- boolean multiple = post != null && post.containsKey("multipleThreaddump");
+ final boolean multiple = post != null && post.containsKey("multipleThreaddump");
if (multiple) {
multipleCount = post.getInt("count", multipleCount);
}
- String threaddump = ThreadDump.threaddump(sb, plain, sleep, multiple, multipleCount);
+ final String threaddump = ThreadDump.threaddump(sb, plain, sleep, multiple, multipleCount);
prop.put("plain_count", multipleCount);
prop.put("plain_content", threaddump);
prop.put("plain", (plain) ? 1 : 0);
diff --git a/htroot/Trails.java b/source/net/yacy/htroot/Trails.java
similarity index 95%
rename from htroot/Trails.java
rename to source/net/yacy/htroot/Trails.java
index 1d0fe663f..77af2d602 100644
--- a/htroot/Trails.java
+++ b/source/net/yacy/htroot/Trails.java
@@ -7,17 +7,19 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
- *
+ *
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
- *
+ *
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
diff --git a/htroot/TransNews_p.java b/source/net/yacy/htroot/TransNews_p.java
similarity index 89%
rename from htroot/TransNews_p.java
rename to source/net/yacy/htroot/TransNews_p.java
index 323a1f004..c0652d855 100644
--- a/htroot/TransNews_p.java
+++ b/source/net/yacy/htroot/TransNews_p.java
@@ -25,6 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
@@ -49,7 +51,7 @@ public class TransNews_p {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
- String currentlang = sb.getConfig("locale.language", "default");
+ final String currentlang = sb.getConfig("locale.language", "default");
prop.put("currentlang", currentlang);
if ("default".equals(currentlang) || "browser".equals(currentlang)) {
@@ -60,38 +62,38 @@ public class TransNews_p {
prop.put("errmsg", 0);
TranslationManager transMgr = new TranslationManager();
- File locallangFile = transMgr.getScratchFile(new File(currentlang + ".lng"));
- Map<String, Map<String, String>> localTrans = transMgr.loadTranslationsLists(locallangFile);
+ final File locallangFile = transMgr.getScratchFile(new File(currentlang + ".lng"));
+ final Map<String, Map<String, String>> localTrans = transMgr.loadTranslationsLists(locallangFile);
// calculate size of local translations list
int size = 0;
- for (Map<String, String> lst : localTrans.values()) {
+ for (final Map<String, String> lst : localTrans.values()) {
size += lst.size();
}
prop.put("transsize", size);
-
+
// read voting
if ((post != null) && post.containsKey("publishtranslation")) {
- Iterator<String> filenameit = localTrans.keySet().iterator();
+ final Iterator<String> filenameit = localTrans.keySet().iterator();
int msgcounter = 0;
while (filenameit.hasNext()) {
- String file = filenameit.next();
- Map<String, String> tmptrans = localTrans.get(file);
- for (String sourcetxt : tmptrans.keySet()) {
- String targettxt = tmptrans.get(sourcetxt);
+ final String file = filenameit.next();
+ final Map<String, String> tmptrans = localTrans.get(file);
+ for (final String sourcetxt : tmptrans.keySet()) {
+ final String targettxt = tmptrans.get(sourcetxt);
if (targettxt != null && !targettxt.isEmpty()) {
boolean sendit = true;
// check if already published (in newsPool)
- Iterator<NewsDB.Record> it = sb.peers.newsPool.recordIterator(NewsPool.INCOMING_DB);
+ final Iterator<NewsDB.Record> it = sb.peers.newsPool.recordIterator(NewsPool.INCOMING_DB);
while (it.hasNext()) {
- NewsDB.Record rtmp = it.next();
+ final NewsDB.Record rtmp = it.next();
if (rtmp == null) {
continue;
}
if (NewsPool.CATEGORY_TRANSLATION_ADD.equals(rtmp.category())) {
- String tmplng = rtmp.attribute("language", null);
- String tmpfile = rtmp.attribute("file", null);
- String tmpsource = rtmp.attribute("source", null);
+ final String tmplng = rtmp.attribute("language", null);
+ final String tmpfile = rtmp.attribute("file", null);
+ final String tmpsource = rtmp.attribute("source", null);
//String tmptarget = rtmp.attribute("target", null);
// if news with file and source exist (maybe from other peer) - skip sending another msg (to avoid confusion)
@@ -135,12 +137,12 @@ public class TransNews_p {
}
}
}
-
+
if ((post != null) && ((refid = post.get("votePositive", null)) != null)) {
final String filename = post.get("filename");
- File lngfile = new File(sb.getAppPath("locale.source", "locales"), currentlang + ".lng");
+ final File lngfile = new File(sb.getAppPath("locale.source", "locales"), currentlang + ".lng");
transMgr = new TranslationManager(lngfile); // load full language for check if entry is new (globally)
if (transMgr.addTranslation(filename, post.get("source"), post.get("target"))) {
// add to local translation extension
@@ -162,7 +164,7 @@ public class TransNews_p {
sb.peers.newsPool.moveOff(NewsPool.INCOMING_DB, refid);
} catch (IOException | SpaceExceededException ex) {
}
-
+
}
// create Translation voting list
@@ -182,7 +184,7 @@ public class TransNews_p {
String target;
while (k.hasNext()) {
-
+
refid = k.next();
if (refid == null) {
continue;
@@ -193,7 +195,7 @@ public class TransNews_p {
continue;
}
- String lang = row.attribute("language", null);
+ final String lang = row.attribute("language", null);
filename = row.attribute("file", null);
source = row.attribute("source", null);
target = row.attribute("target", null);
@@ -202,14 +204,14 @@ public class TransNews_p {
}
if (!lang.equals(currentlang)) continue;
-
+
String existingtarget = null; //transMgr.getTranslation(filename, source);
- Map<String, String> tmpMap = localTrans.get(filename);
+ final Map<String, String> tmpMap = localTrans.get(filename);
if (tmpMap != null) {
existingtarget = tmpMap.get(source);
}
- boolean altexist = existingtarget != null && !target.isEmpty() && !existingtarget.isEmpty() && !existingtarget.equals(target);
+ final boolean altexist = existingtarget != null && !target.isEmpty() && !existingtarget.isEmpty() && !existingtarget.equals(target);
prop.put("results_" + i + "_refid", refid);
prop.put("results_" + i + "_url", filename); // url to local file
@@ -280,7 +282,7 @@ public class TransNews_p {
String filename="";
String source="";
String target="";
-
+
int score = 0;
Integer vote;
@@ -304,7 +306,7 @@ public class TransNews_p {
}
// add/subtract votes and write record
-
+
if ((vote = negativeHashes.get(refid)) != null) {
score -= vote.intValue();
}
diff --git a/htroot/Translator_p.java b/source/net/yacy/htroot/Translator_p.java
similarity index 85%
rename from htroot/Translator_p.java
rename to source/net/yacy/htroot/Translator_p.java
index e8af97b26..8a7906a4e 100644
--- a/htroot/Translator_p.java
+++ b/source/net/yacy/htroot/Translator_p.java
@@ -18,10 +18,13 @@
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.TransactionManager;
@@ -39,7 +42,7 @@ public class Translator_p {
final servletProperties prop = new servletProperties();
final Switchboard sb = (Switchboard) env;
- String langcfg = sb.getConfig("locale.language", "default");
+ final String langcfg = sb.getConfig("locale.language", "default");
prop.put("targetlang", langcfg);
if ("default".equals(langcfg) || "browser".equals(langcfg)) {
prop.put("errmsg", 1); // msg: activate diff lng
@@ -48,14 +51,14 @@ public class Translator_p {
prop.put("errmsg", 0);
final File localesFolder = sb.getAppPath("locale.source", "locales");
- File lngfile = new File(localesFolder, langcfg + ".lng");
- TranslationManager localTransMgr = new TranslationManager(/*new File ("locales","master.lng.xlf")*/);
+ final File lngfile = new File(localesFolder, langcfg + ".lng");
+ final TranslationManager localTransMgr = new TranslationManager(/*new File ("locales","master.lng.xlf")*/);
- File masterxlf = new File(localesFolder, "master.lng.xlf");
+ final File masterxlf = new File(localesFolder, "master.lng.xlf");
if (!masterxlf.exists()) localTransMgr.createMasterTranslationLists(localesFolder, masterxlf);
- Map<String, Map<String, String>> origTrans = localTransMgr.joinMasterTranslationLists(masterxlf, lngfile);
+ final Map<String, Map<String, String>> origTrans = localTransMgr.joinMasterTranslationLists(masterxlf, lngfile);
final File locallngfile = localTransMgr.getScratchFile(lngfile);
- Map<String, Map<String, String>> localTrans = localTransMgr.loadTranslationsLists(locallngfile); // TODO: this will read file twice
+ final Map<String, Map<String, String>> localTrans = localTransMgr.loadTranslationsLists(locallngfile); // TODO: this will read file twice
int i = 0;
if (origTrans.size() > 0) {
String filename = origTrans.keySet().iterator().next();
@@ -64,14 +67,14 @@ public class Translator_p {
}
// prepare filename list for drop-down list
- Iterator<String> filenameit = origTrans.keySet().iterator();
+ final Iterator<String> filenameit = origTrans.keySet().iterator();
final boolean filteruntranslated = post == null ? false : post.getBoolean("filteruntranslated");
while (filenameit.hasNext()) {
- String tmp = filenameit.next();
+ final String tmp = filenameit.next();
if (filteruntranslated) { // filter filenames with untranslated entries
- Map<String, String> tmptrans = origTrans.get(tmp);
+ final Map<String, String> tmptrans = origTrans.get(tmp);
boolean hasuntrans = false;
- for (String x : tmptrans.values()) {
+ for (final String x : tmptrans.values()) {
if (x == null || x.isEmpty()) {
hasuntrans = true;
break;
@@ -86,7 +89,7 @@ public class Translator_p {
prop.put("filelist", i);
prop.add("sourcefile", filename);
- Map<String, String> origTextList = origTrans.get(filename);
+ final Map<String, String> origTextList = origTrans.get(filename);
i = 0;
boolean editapproved = false; // to switch already approved translation in edit mode
@@ -100,12 +103,12 @@ public class Translator_p {
}
}
boolean changed = false;
- for (String sourcetext : origTextList.keySet()) {
+ for (final String sourcetext : origTextList.keySet()) {
String targettxt = origTextList.get(sourcetext);
boolean existinlocalTrans = localTrans.containsKey(filename) && localTrans.get(filename).containsKey(sourcetext);
-
+
if (editapproved) existinlocalTrans |= (i == textlistid); // if edit of approved requested handle as/like existing in local to allow edit
-
+
if (targettxt == null || targettxt.isEmpty() || existinlocalTrans) {
prop.put("textlist_" + i + "_filteruntranslated", true);
} else if (filteruntranslated) {
@@ -115,7 +118,7 @@ public class Translator_p {
if (i == textlistid && post != null) {
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(requestHeader, post);
-
+
if (editapproved) { // switch already translated in edit mode by copying to local translation
// not saved here as not yet modified/approved
localTransMgr.addTranslation(localTrans, filename, sourcetext, targettxt);
@@ -144,7 +147,7 @@ public class Translator_p {
if (changed) {
/* Check this is a valid transaction */
TransactionManager.checkPostTransaction(requestHeader, post);
-
+
localTransMgr.saveAsLngFile(langcfg, locallngfile, localTrans);
// adhoc translate this file
// 1. get/calc the path
@@ -157,13 +160,13 @@ public class Translator_p {
localTransMgr.translateFile(sourceFile, destFile, origTextList); // do the translation
}
}
-
+
/* Acquire a transaction token for the next POST form submission */
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(requestHeader));
-
+
prop.put("textlist", i);
return prop;
- } catch (IOException ex) {
+ } catch (final IOException ex) {
ConcurrentLog.logException(ex);
}
return null;
diff --git a/htroot/User.java b/source/net/yacy/htroot/User.java
similarity index 98%
rename from htroot/User.java
rename to source/net/yacy/htroot/User.java
index 5f6fb9daa..5c8e3f130 100644
--- a/htroot/User.java
+++ b/source/net/yacy/htroot/User.java
@@ -27,7 +27,10 @@
//javac -classpath .:../Classes Message.java
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import javax.servlet.ServletException;
+
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.order.Digest;
import net.yacy.cora.protocol.RequestHeader;
@@ -163,7 +166,7 @@ public class User{
}
try {
requestHeader.logout(); // servlet container session logout
- } catch (ServletException ex) {}
+ } catch (final ServletException ex) {}
if(post.containsKey("returnto")){
prop.putHTML(serverObjects.ACTION_LOCATION, post.get("returnto"));
}
diff --git a/htroot/ViewFavicon.java b/source/net/yacy/htroot/ViewFavicon.java
similarity index 92%
rename from htroot/ViewFavicon.java
rename to source/net/yacy/htroot/ViewFavicon.java
index 492a5cb3a..755aa81f0 100644
--- a/htroot/ViewFavicon.java
+++ b/source/net/yacy/htroot/ViewFavicon.java
@@ -18,6 +18,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
@@ -45,12 +47,12 @@ import net.yacy.visualization.ImageViewer;
/**
* Extends ViewImage behavior : add a specific favicon cache and use of a
* default image on loading error.
- *
+ *
* @author luc
*
*/
public class ViewFavicon {
-
+
/** Single instance of ImageViewer */
private static final ImageViewer VIEWER = new ImageViewer();
@@ -72,7 +74,7 @@ public class ViewFavicon {
* "code" parameter (non authenticated users). When image could be parsed,
* try encoding to target format specified by header "EXT". When any error
* occurs, return default icon.
- *
+ *
* @param header
* request header
* @param post
@@ -85,8 +87,8 @@ public class ViewFavicon {
public static Object respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
- String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, null);
- boolean isPNGTarget = "png".equalsIgnoreCase(ext);
+ final String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, null);
+ final boolean isPNGTarget = "png".equalsIgnoreCase(ext);
ImageInputStream imageInStream = null;
InputStream inStream = null;
@@ -99,7 +101,7 @@ public class ViewFavicon {
final boolean auth = ImageViewer.hasFullViewingRights(header, sb); // handle access rights
- DigestURL url = VIEWER.parseURL(post, auth);
+ final DigestURL url = VIEWER.parseURL(post, auth);
final String normalizedURL = url.toNormalform(false);
@@ -108,7 +110,7 @@ public class ViewFavicon {
}
/* Icon is not already in cache */
if (resultBytes == null) {
- String urlExt = MultiProtocolURL.getFileExtension(url.getFileName());
+ final String urlExt = MultiProtocolURL.getFileExtension(url.getFileName());
if (ext != null && ext.equalsIgnoreCase(urlExt) && ImageViewer.isBrowserRendered(urlExt)) {
return VIEWER.openInputStream(post, sb.loader, auth, url);
}
@@ -123,7 +125,7 @@ public class ViewFavicon {
imageInStream = ImageIO.createImageInputStream(inStream);
}
// read image
- EncodedImage encodedIcon = VIEWER.parseAndScale(post, auth, url, ext, imageInStream);
+ final EncodedImage encodedIcon = VIEWER.parseAndScale(post, auth, url, ext, imageInStream);
if (encodedIcon != null && !encodedIcon.getImage().isEmpty()) {
resultBytes = encodedIcon.getImage().getBytes();
@@ -133,7 +135,7 @@ public class ViewFavicon {
}
}
}
- } catch (IOException e) {
+ } catch (final IOException e) {
ConcurrentLog.fine("ViewFavicon", "Error loading favicon, default one wille be used : " + e);
} finally {
/*
@@ -142,7 +144,7 @@ public class ViewFavicon {
if (inStream != null) {
try {
inStream.close();
- } catch (IOException ignored) {
+ } catch (final IOException ignored) {
}
}
if (resultBytes == null) {
@@ -167,7 +169,7 @@ public class ViewFavicon {
/**
* Load default icon and encode it to ext format
- *
+ *
* @param post
* post parameters
* @param sb
@@ -177,7 +179,7 @@ public class ViewFavicon {
* @return icon encoded bytes, empty if and exception occured when loading
* or rendering
*/
- private static byte[] loadDefaultIcon(final serverObjects post, final Switchboard sb, String ext) {
+ private static byte[] loadDefaultIcon(final serverObjects post, final Switchboard sb, final String ext) {
byte[] resultBytes;
byte[] defaultBytes = new byte[0];
try {
diff --git a/htroot/ViewFile.java b/source/net/yacy/htroot/ViewFile.java
similarity index 96%
rename from htroot/ViewFile.java
rename to source/net/yacy/htroot/ViewFile.java
index e8a4de344..bbf1ba1ee 100644
--- a/htroot/ViewFile.java
+++ b/source/net/yacy/htroot/ViewFile.java
@@ -27,6 +27,8 @@
//javac -classpath .:../Classes Status.java
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.charset.Charset;
@@ -95,11 +97,11 @@ public class ViewFile {
prop.put("error", "1");
prop.put("url", "");
prop.put("showSnippet", 0);
-
+
if (post == null) return prop;
// get segment
- Segment indexSegment = sb.index;
+ final Segment indexSegment = sb.index;
final boolean authorized = sb.verifyAuthentication(header);
if (post.containsKey("words"))
@@ -120,7 +122,7 @@ public class ViewFile {
final String viewMode = showSnippet ? "sentences" : post.get("viewMode", "sentences");
prop.put("error_vMode-" + viewMode, "1");
prop.put("viewModeValue", viewMode);
-
+
DigestURL url = null;
String descr = "";
final int wordCount = 0;
@@ -216,9 +218,9 @@ public class ViewFile {
/* Encoding is known but not supported on this system : default decode using UTF-8 */
charsetName = StandardCharsets.UTF_8.name();
}
- } catch(IllegalCharsetNameException e) {
+ } catch(final IllegalCharsetNameException e) {
/* Encoding is known but charset name is not valid : default decode using UTF-8 */
- charsetName = StandardCharsets.UTF_8.name();
+ charsetName = StandardCharsets.UTF_8.name();
}
content = new String(response.getContent(), charsetName);
} catch (final Exception e) {
@@ -346,8 +348,8 @@ public class ViewFile {
}
// optional: generate snippet
if (showSnippet) {
- QueryGoal goal = new QueryGoal(post.get("search", ""));
- TextSnippet snippet = new TextSnippet(
+ final QueryGoal goal = new QueryGoal(post.get("search", ""));
+ final TextSnippet snippet = new TextSnippet(
sb.loader,
urlEntry,
goal.getIncludeWordsSet(),
@@ -383,7 +385,7 @@ public class ViewFile {
prop.put("error_inurldb_hash", urlHash);
prop.put("error_wordCount", wordCount);
prop.put("error_firstSeen", "");
- long firstseen = sb.index.getFirstSeenTime(ASCII.getBytes(urlHash));
+ final long firstseen = sb.index.getFirstSeenTime(ASCII.getBytes(urlHash));
prop.put("error_firstSeen", firstseen < 0 ? "" : new Date(firstseen).toString());
prop.putHTML("error_desc", (descr.isEmpty()) ? "&nbsp;" : descr);
prop.putNum("error_size", size);
@@ -566,7 +568,7 @@ public class ViewFile {
}
return i;
}
-
+
/**
* Fill prop object with icon links.
* @param prop object ot be filled
diff --git a/htroot/ViewImage.java b/source/net/yacy/htroot/ViewImage.java
similarity index 92%
rename from htroot/ViewImage.java
rename to source/net/yacy/htroot/ViewImage.java
index 1db8970eb..1f3169c74 100644
--- a/htroot/ViewImage.java
+++ b/source/net/yacy/htroot/ViewImage.java
@@ -21,6 +21,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.IOException;
import java.io.InputStream;
@@ -39,7 +41,7 @@ import net.yacy.server.serverSwitch;
import net.yacy.visualization.ImageViewer;
public class ViewImage {
-
+
/** Single instance of ImageViewer */
private static final ImageViewer VIEWER = new ImageViewer();
@@ -48,7 +50,7 @@ public class ViewImage {
* When image format is not supported, return directly image data. When
* image could be parsed, try encoding to target format specified by header
* "EXT".
- *
+ *
* @param header
* request header
* @param post
@@ -56,9 +58,9 @@ public class ViewImage {
* @param env
* environment
* @return an {@link EncodedImage} instance encoded in format specified in
- * post, or an InputStream pointing to original image data.
- * Return and EncodedImage with empty data when image format is not supported,
- * a read/write or any other error occured while loading resource.
+ * post, or an InputStream pointing to original image data.
+ * Return and EncodedImage with empty data when image format is not supported,
+ * a read/write or any other error occured while loading resource.
* @throws IOException
* when specified url is malformed.
* Sould end in a HTTP 500 error whose processing is more
@@ -75,10 +77,10 @@ public class ViewImage {
throw new TemplateMissingParameterException("please fill at least url or code parameter");
}
- String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, null);
+ final String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, null);
final boolean auth = ImageViewer.hasFullViewingRights(header, sb); // handle access rights
- DigestURL url = VIEWER.parseURL(post, auth);
+ final DigestURL url = VIEWER.parseURL(post, auth);
// get the image as stream
EncodedImage encodedImage;
@@ -86,7 +88,7 @@ public class ViewImage {
ImageInputStream imageInStream = null;
InputStream inStream = null;
try {
- String urlExt = MultiProtocolURL.getFileExtension(url.getFileName());
+ final String urlExt = MultiProtocolURL.getFileExtension(url.getFileName());
if (ext != null && ext.equalsIgnoreCase(urlExt) && ImageViewer.isBrowserRendered(urlExt)) {
return VIEWER.openInputStream(post, sb.loader, auth, url);
}
@@ -102,7 +104,7 @@ public class ViewImage {
}
// read image
encodedImage = VIEWER.parseAndScale(post, auth, url, ext, imageInStream);
- } catch (Exception e) {
+ } catch (final Exception e) {
/*
* Exceptions are not propagated here : many error causes are
* possible, network errors, incorrect or unsupported format, bad
@@ -118,7 +120,7 @@ public class ViewImage {
if (inStream != null) {
try {
inStream.close();
- } catch (IOException ignored) {
+ } catch (final IOException ignored) {
}
}
}
diff --git a/htroot/ViewLog_p.java b/source/net/yacy/htroot/ViewLog_p.java
similarity index 96%
rename from htroot/ViewLog_p.java
rename to source/net/yacy/htroot/ViewLog_p.java
index a5783226e..f0a2df9d2 100644
--- a/htroot/ViewLog_p.java
+++ b/source/net/yacy/htroot/ViewLog_p.java
@@ -30,6 +30,8 @@
//javac -classpath .:../classes ViewLog_p.java
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.logging.Handler;
import java.util.logging.Logger;
import java.util.regex.Matcher;
diff --git a/htroot/ViewProfile.java b/source/net/yacy/htroot/ViewProfile.java
similarity index 95%
rename from htroot/ViewProfile.java
rename to source/net/yacy/htroot/ViewProfile.java
index e1b1750b1..ab99b3f92 100644
--- a/htroot/ViewProfile.java
+++ b/source/net/yacy/htroot/ViewProfile.java
@@ -31,6 +31,8 @@
// javac -classpath .:../Classes Blacklist_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -67,7 +69,7 @@ public class ViewProfile {
env.getConfig("network.unit.description", "") :
env.getConfig(SwitchboardConstants.GREETING, "");
prop.put("display_promoteSearchPageGreeting", promoteSearchPageGreeting);
-
+
prop.put("edit", authenticated ? 1 : 0);
final String hash = (post == null) ? null : post.get("hash");
@@ -91,7 +93,7 @@ public class ViewProfile {
if (fileIn != null) try { fileIn.close(); fileIn = null; } catch (final Exception e) {}
}
profile = new HashMap<String, String>();
- for (Map.Entry<Object, Object> e: p.entrySet()) profile.put((String) e.getKey(), (String) e.getValue());
+ for (final Map.Entry<Object, Object> e: p.entrySet()) profile.put((String) e.getKey(), (String) e.getValue());
prop.put("success", "3"); // everything ok
prop.put("localremotepeer", "0");
prop.putHTML("success_peername", sb.peers.mySeed().getName());
diff --git a/htroot/Vocabulary_p.java b/source/net/yacy/htroot/Vocabulary_p.java
similarity index 90%
rename from htroot/Vocabulary_p.java
rename to source/net/yacy/htroot/Vocabulary_p.java
index 5944c0fa0..614964eb0 100644
--- a/htroot/Vocabulary_p.java
+++ b/source/net/yacy/htroot/Vocabulary_p.java
@@ -18,6 +18,8 @@
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
@@ -65,24 +67,24 @@ import net.yacy.server.serverSwitch;
* Handle creation and edition of vocabularies through the Vocabulary_p.html page.
*/
public class Vocabulary_p {
-
+
/** Logger */
private final static ConcurrentLog LOG = new ConcurrentLog(Vocabulary_p.class.getSimpleName());
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
-
+
/* Acquire a transaction token for the next POST form submission */
final String nextToken = TransactionManager.getTransactionToken(header);
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, nextToken);
prop.put("edit_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextToken);
prop.put("create_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextToken);
-
- Collection<Tagging> vocs = LibraryProvider.autotagging.getVocabularies();
+
+ final Collection<Tagging> vocs = LibraryProvider.autotagging.getVocabularies();
String vocabularyName = (post == null) ? null : post.get("vocabulary", null);
- String discovername = (post == null) ? null : post.get("discovername", null);
+ final String discovername = (post == null) ? null : post.get("discovername", null);
Tagging vocabulary = vocabularyName == null ? null : LibraryProvider.autotagging.getVocabulary(vocabularyName);
if (vocabulary == null) {
vocabularyName = null;
@@ -92,7 +94,7 @@ public class Vocabulary_p {
if (vocabulary == null && discovername != null && discovername.length() > 0) {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
-
+
// get details of creation
String discoverobjectspace = post.get("discoverobjectspace", "");
MultiProtocolURL discoveruri = null;
@@ -123,7 +125,7 @@ public class Vocabulary_p {
csvUrl = new DigestURL(discoverFromCSVPath);
} catch(final MalformedURLException e) {
csvFileStatus = 5;
- prop.put("create_csvFileStatus_csvUrl", discoverFromCSVPath);
+ prop.put("create_csvFileStatus_csvUrl", discoverFromCSVPath);
}
} else {
final File discoverFromCSVFile = new File(discoverFromCSVPath);
@@ -142,31 +144,31 @@ public class Vocabulary_p {
csvUrl = new DigestURL(discoverFromCSVFile);
} catch(final MalformedURLException e) {
csvFileStatus = 5;
- prop.put("create_csvFileStatus_csvUrl", "file://" + discoverFromCSVFile.getAbsolutePath());
+ prop.put("create_csvFileStatus_csvUrl", "file://" + discoverFromCSVFile.getAbsolutePath());
}
}
}
-
+
if(csvUrl != null) {
try {
handleDiscoverFromCSV(sb, post, table, csvUrl);
} catch(final IOException e) {
LOG.warn("Could not read CSV file at " + csvUrl, e);
csvFileStatus = 3;
- prop.put("create_csvFileStatus_csvFile", csvUrl.toString());
+ prop.put("create_csvFileStatus_csvFile", csvUrl.toString());
}
}
}
} else {
- Iterator<DigestURL> ui = segment.urlSelector(discoveruri, Long.MAX_VALUE, 100000);
+ final Iterator<DigestURL> ui = segment.urlSelector(discoveruri, Long.MAX_VALUE, 100000);
while (ui.hasNext()) {
- DigestURL u = ui.next();
- String u0 = u.toNormalform(true);
+ final DigestURL u = ui.next();
+ final String u0 = u.toNormalform(true);
t = "";
if (discoverFromPath) {
- int exp = u0.lastIndexOf('.');
+ final int exp = u0.lastIndexOf('.');
if (exp < 0) continue;
- int slp = u0.lastIndexOf('/', exp);
+ final int slp = u0.lastIndexOf('/', exp);
if (slp < 0) continue;
t = u0.substring(slp, exp);
int p;
@@ -174,28 +176,28 @@ public class Vocabulary_p {
while ((p = t.indexOf('=')) >= 0) t = t.substring(p + 1);
}
if (discoverFromTitle || discoverFromTitleSplitted) {
- URIMetadataNode m = segment.fulltext().getMetadata(u.hash());
+ final URIMetadataNode m = segment.fulltext().getMetadata(u.hash());
if (m != null) t = m.dc_title();
if (t.endsWith(".jpg") || t.endsWith(".gif")) continue;
}
if (discoverFromAuthor) {
- URIMetadataNode m = segment.fulltext().getMetadata(u.hash());
+ final URIMetadataNode m = segment.fulltext().getMetadata(u.hash());
if (m != null) t = m.dc_creator();
}
t = t.replaceAll("_", " ").replaceAll("\"", " ").replaceAll("'", " ").replaceAll(",", " ").replaceAll(" ", " ").trim();
if (t.isEmpty()) continue;
if (discoverFromTitleSplitted) {
- String[] ts = CommonPattern.SPACES.split(t);
- for (String s: ts) {
+ final String[] ts = CommonPattern.SPACES.split(t);
+ for (final String s: ts) {
if (s.isEmpty()) continue;
if (s.endsWith(".jpg") || s.endsWith(".gif")) continue;
table.put(s, new Tagging.SOTuple(Tagging.normalizeTerm(s), u0));
}
} else if (discoverFromAuthor) {
- String[] ts = CommonPattern.SEMICOLON.split(t); // author names are often separated by ';'
+ final String[] ts = CommonPattern.SEMICOLON.split(t); // author names are often separated by ';'
for (String s: ts) {
if (s.isEmpty()) continue;
- int p = s.indexOf(','); // check if there is a reversed method to mention the name
+ final int p = s.indexOf(','); // check if there is a reversed method to mention the name
if (p >= 0) s = s.substring(p + 1).trim() + " " + s.substring(0, p).trim();
table.put(s, new Tagging.SOTuple(Tagging.normalizeTerm(s), u0));
}
@@ -208,13 +210,13 @@ public class Vocabulary_p {
prop.put("create_csvFileStatus", csvFileStatus);
if(csvFileStatus == 0) {
try {
- Tagging newvoc = new Tagging(discovername, propFile, discoverobjectspace, table);
+ final Tagging newvoc = new Tagging(discovername, propFile, discoverobjectspace, table);
prop.put("create_vocabWriteError", false);
-
+
LibraryProvider.autotagging.addVocabulary(newvoc);
vocabularyName = discovername;
vocabulary = newvoc;
-
+
// store this call as api call
sb.tables.recordAPICall(post, "Vocabulary_p.html", WorkTables.TABLE_API_TYPE_CRAWLER, "vocabulary creation for " + discovername);
} catch(final IOException e) {
@@ -227,7 +229,7 @@ public class Vocabulary_p {
} else if (vocabulary != null && post.containsKey("set")) {
/* Check the transaction is valid */
TransactionManager.checkPostTransaction(header, post);
-
+
try {
// check if objectspace was set
vocabulary.setObjectspace(post.get("objectspace", vocabulary.getObjectspace() == null ? "" : vocabulary.getObjectspace()));
@@ -242,11 +244,11 @@ public class Vocabulary_p {
}
// check if a term was modified
- for (Map.Entry<String, String> e : post.entrySet()) {
+ for (final Map.Entry<String, String> e : post.entrySet()) {
if (e.getKey().startsWith("modify_") && e.getValue().equals("checked")) {
- String term = e.getKey().substring(7);
- String synonyms = post.get("synonyms_" + term, "");
- String objectlink = post.get("objectlink_" + term, "");
+ final String term = e.getKey().substring(7);
+ final String synonyms = post.get("synonyms_" + term, "");
+ final String objectlink = post.get("objectlink_" + term, "");
vocabulary.put(term, synonyms, objectlink);
}
}
@@ -264,32 +266,32 @@ public class Vocabulary_p {
}
// check if a term shall be deleted
- if (vocabulary != null && vocabulary.size() > 0) for (Map.Entry<String, String> e : post.entrySet()) {
+ if (vocabulary != null && vocabulary.size() > 0) for (final Map.Entry<String, String> e : post.entrySet()) {
if (e.getKey().startsWith("delete_") && e.getValue().equals("checked")) {
- String term = e.getKey().substring(7);
+ final String term = e.getKey().substring(7);
vocabulary.delete(term);
}
}
-
+
// check the isFacet and isMatchFromLinkedData properties
if (vocabulary != null && post.containsKey("set")) {
- boolean isFacet = post.getBoolean("isFacet");
+ final boolean isFacet = post.getBoolean("isFacet");
vocabulary.setFacet(isFacet);
- Set<String> omit = env.getConfigSet("search.result.show.vocabulary.omit");
+ final Set<String> omit = env.getConfigSet("search.result.show.vocabulary.omit");
if (isFacet) {
- omit.remove(vocabularyName);
+ omit.remove(vocabularyName);
} else {
omit.add(vocabularyName);
}
env.setConfig("search.result.show.vocabulary.omit", omit);
-
- boolean isMatchFromLinkedData = post.getBoolean("vocabularies.matchLinkedData");
+
+ final boolean isMatchFromLinkedData = post.getBoolean("vocabularies.matchLinkedData");
vocabulary.setMatchFromLinkedData(isMatchFromLinkedData);
final Set<String> matchLinkedDataVocs = env.getConfigSet(SwitchboardConstants.VOCABULARIES_MATCH_LINKED_DATA_NAMES);
if (isMatchFromLinkedData) {
matchLinkedDataVocs.add(vocabularyName);
} else {
- matchLinkedDataVocs.remove(vocabularyName);
+ matchLinkedDataVocs.remove(vocabularyName);
}
env.setConfig(SwitchboardConstants.VOCABULARIES_MATCH_LINKED_DATA_NAMES, matchLinkedDataVocs);
}
@@ -301,7 +303,7 @@ public class Vocabulary_p {
}
int count = 0;
- for (Tagging v: vocs) {
+ for (final Tagging v: vocs) {
prop.put("vocabularyset_" + count + "_name", v.getName());
prop.put("vocabularyset_" + count + "_selected", ((vocabularyName != null && vocabularyName.equals(v.getName())) || (discovername != null && discovername.equals(v.getName()))) ? 1 : 0);
count++;
@@ -314,7 +316,7 @@ public class Vocabulary_p {
prop.put("edit", 0);
} else {
prop.put("edit", 1);
- boolean editable = vocabulary.getFile() != null && vocabulary.getFile().exists();
+ final boolean editable = vocabulary.getFile() != null && vocabulary.getFile().exists();
prop.put("edit_editable", editable ? 1 : 0);
prop.putHTML("edit_editable_file", editable ? vocabulary.getFile().getAbsolutePath() : "");
prop.putHTML("edit_name", vocabulary.getName());
@@ -329,20 +331,20 @@ public class Vocabulary_p {
prop.putHTML("edit_editable_objectspacepredicate", DCTerms.references.getPredicate());
int c = 0;
boolean dark = false;
- int osl = vocabulary.getObjectspace() == null ? 0 : vocabulary.getObjectspace().length();
- Map<String, SOTuple> list = vocabulary.list();
+ final int osl = vocabulary.getObjectspace() == null ? 0 : vocabulary.getObjectspace().length();
+ final Map<String, SOTuple> list = vocabulary.list();
prop.put("edit_size", list.size());
- for (Map.Entry<String, SOTuple> entry: list.entrySet()) {
+ for (final Map.Entry<String, SOTuple> entry: list.entrySet()) {
prop.put("edit_terms_" + c + "_editable", editable ? 1 : 0);
prop.put("edit_terms_" + c + "_dark", dark ? 1 : 0); dark = !dark;
prop.putXML("edit_terms_" + c + "_label", osl > entry.getValue().getObjectlink().length() ? entry.getKey() : entry.getValue().getObjectlink().substring(osl));
prop.putHTML("edit_terms_" + c + "_term", entry.getKey());
prop.putXML("edit_terms_" + c + "_termxml", entry.getKey());
prop.putHTML("edit_terms_" + c + "_editable_term", entry.getKey());
- String synonymss = entry.getValue().getSynonymsCSV();
+ final String synonymss = entry.getValue().getSynonymsCSV();
prop.putHTML("edit_terms_" + c + "_editable_synonyms", synonymss);
if (synonymss.length() > 0) {
- String[] synonymsa = entry.getValue().getSynonymsList();
+ final String[] synonymsa = entry.getValue().getSynonymsList();
for (int i = 0; i < synonymsa.length; i++) {
prop.put("edit_terms_" + c + "_synonyms_" + i + "_altLabel", synonymsa[i]);
}
@@ -362,22 +364,22 @@ public class Vocabulary_p {
prop.putHTML("create_charset_" + 0 + "_name", "autodetect");
prop.put("create_charset_" + 0 + "_selected", 1);
int c = 1;
- for (String cs: Charset.availableCharsets().keySet()) {
+ for (final String cs: Charset.availableCharsets().keySet()) {
prop.putHTML("create_charset_" + c + "_name", cs);
prop.put("create_charset_" + c + "_selected", 0);
c++;
}
prop.put("create_charset", c);
-
+
// return rewrite properties
return prop;
}
-
+
/**
* Parse a CSV content line and extract field values. When the last field of
* this line starts with and unclosed escape character, the current line is
* appended to the escapedContent buffer.
- *
+ *
* @param line
* a raw line from a CSV document. Must not be null.
* @param separatorPattern
@@ -396,7 +398,7 @@ public class Vocabulary_p {
if (tokens.length == 0) {
tokens = new String[]{line};
}
-
+
/* Handle continuation of multi-lines field content escaped between escape char */
if(multiLineContent.length() > 0) {
int closingEscapeIndex = -1;
@@ -420,7 +422,7 @@ public class Vocabulary_p {
return fields;
}
}
-
+
/* Handle separator char escaped between escape char */
final StringBuilder escapedSeparatorContent = new StringBuilder();
for(final String field : tokens) {
@@ -443,7 +445,7 @@ public class Vocabulary_p {
}
fields.add(field);
}
-
+
if(escapedSeparatorContent.length() > 0) {
/* Handle beginning of field content with escaped line separator */
multiLineContent.setLength(0);
@@ -474,9 +476,9 @@ public class Vocabulary_p {
final int discovercolumnobjectlink = post.getInt("discovercolumnobjectlink", -1);
final boolean discoverenrichsynonyms = post.get("discoversynonymsmethod", "none").equals("enrichsynonyms");
final boolean discoverreadcolumn = post.get("discoversynonymsmethod", "none").equals("readcolumn");
-
+
final Pattern separatorPattern = Pattern.compile(columnSeparator);
-
+
// auto-detect charset, used code from http://jchardet.sourceforge.net/; see also: http://www-archive.mozilla.org/projects/intl/chardet.html
if (charsetName.equals("autodetect")) {
@@ -486,9 +488,9 @@ public class Vocabulary_p {
if(streamResponse == null || streamResponse.getContentStream() == null) {
throw new IOException("Could not get CSV content at " + csvFileUrl);
}
-
+
charsetName = streamResponse.getResponse().getCharacterEncoding();
-
+
if(charsetName == null) {
/* Charset not provided in response headers : try to detect it from content */
final List<String> charsets = FileUtils.detectCharset(streamResponse.getContentStream());
@@ -507,7 +509,7 @@ public class Vocabulary_p {
}
}
}
-
+
// when autodetection of content charset has been selected, a remote resource may opened again, but has some chances to be now in cache
try(final StreamResponse streamResponse = sb.loader.openInputStream(
sb.loader.request(csvFileUrl, true, false), CacheStrategy.IFFRESH, BlacklistType.CRAWLER,
@@ -527,8 +529,8 @@ public class Vocabulary_p {
/**
* Fill the vocabulary table from reader open on CSV content.
* @param table the vocabulary table to fill. Must not be null.
- * @param escapeChar CSV escape character (standard is double quote). Must not be null.
- * @param lineStart index (zero based) of the first line to parse. Previous lines are ignored.
+ * @param escapeChar CSV escape character (standard is double quote). Must not be null.
+ * @param lineStart index (zero based) of the first line to parse. Previous lines are ignored.
* @param literalsColumn index (zero based) of the column to read for literals
* @param synonymsColumn index (zero based) of the column to read for synonyms. Set to -1 to ignore.
* @param discovercolumnobjectlink
@@ -554,13 +556,13 @@ public class Vocabulary_p {
if (line.length() == 0) {
continue;
}
-
+
final List<String> fields = parseCSVLine(line, separatorPattern, escapeChar, multiLineContent);
if (multiLineContent.length() > 0) {
continue;
}
-
-
+
+
String literal = literalsColumn < 0 || fields.size() <= literalsColumn ? null : fields.get(literalsColumn).trim();
if (literal == null) {
continue;
@@ -584,11 +586,11 @@ public class Vocabulary_p {
}
// check double synonyms
if (synonyms.length() > 0) {
- String oldliteral = synonym2literal.get(synonyms);
+ final String oldliteral = synonym2literal.get(synonyms);
if (oldliteral != null && !literal.equals(oldliteral)) {
// replace old entry with combined new
table.remove(oldliteral);
- String newliteral = oldliteral + "," + literal;
+ final String newliteral = oldliteral + "," + literal;
literal = newliteral;
}
synonym2literal.put(synonyms, literal);
@@ -598,7 +600,7 @@ public class Vocabulary_p {
}
}
}
-
+
private static String normalizeLiteral(String literal, final String escapeChar) {
if (literal == null) {
return "";
diff --git a/htroot/WatchWebStructure_p.java b/source/net/yacy/htroot/WatchWebStructure_p.java
similarity index 95%
rename from htroot/WatchWebStructure_p.java
rename to source/net/yacy/htroot/WatchWebStructure_p.java
index 9a7fa087a..fa1f8b5bd 100644
--- a/htroot/WatchWebStructure_p.java
+++ b/source/net/yacy/htroot/WatchWebStructure_p.java
@@ -4,6 +4,8 @@
//$LastChangedBy$
//
+package net.yacy.htroot;
+
import java.util.Iterator;
import net.yacy.cora.document.id.DigestURL;
@@ -68,12 +70,12 @@ public class WatchWebStructure_p {
host = "www." + host;
}
}
-
+
if (post != null && post.containsKey("hosts")) {
- int maxcount = 200;
- ReversibleScoreMap<String> score = sb.webStructure.hostReferenceScore();
+ final int maxcount = 200;
+ final ReversibleScoreMap<String> score = sb.webStructure.hostReferenceScore();
int c = 0;
- Iterator<String> i = score.keys(false);
+ final Iterator<String> i = score.keys(false);
String h;
while (i.hasNext() && c < maxcount) {
h = i.next();
diff --git a/htroot/WebStructurePicture_p.java b/source/net/yacy/htroot/WebStructurePicture_p.java
similarity index 87%
rename from htroot/WebStructurePicture_p.java
rename to source/net/yacy/htroot/WebStructurePicture_p.java
index 8efa8ace8..7e5ea9e38 100644
--- a/htroot/WebStructurePicture_p.java
+++ b/source/net/yacy/htroot/WebStructurePicture_p.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
@@ -88,7 +90,7 @@ public class WebStructurePicture_p {
color_lineend = post.get("colorlineend", color_lineend);
cyc = post.getInt("cyc", 0);
}
-
+
// calculate target time
final long timeout = (time < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + (time * 8 / 10);
@@ -107,9 +109,9 @@ public class WebStructurePicture_p {
} else {
// recursively find domains, up to a specific depth
GraphPlotter graph = new GraphPlotter();
- String[] hostlist = CommonPattern.COMMA.split(hosts);
+ final String[] hostlist = CommonPattern.COMMA.split(hosts);
for (int i = 0; i < hostlist.length; i++) {
- String host = hostlist[i];
+ final String host = hostlist[i];
double angle = 2.0d * i * Math.PI / hostlist.length;
if (hostlist.length == 3) angle -= Math.PI / 2;
if (hostlist.length == 4) angle += Math.PI / 4;
@@ -120,9 +122,9 @@ public class WebStructurePicture_p {
// apply physics to it to get a better shape
if (post != null && post.containsKey("pa")) {
// test with: http://localhost:8090/WebStructurePicture_p.png?pa=1&ral=0.7&raa=0.5&rar=2&rel=0.5&rea=1&rer=2
- GraphPlotter.Ribbon rAll = new GraphPlotter.Ribbon(post.getFloat("ral", 0.1f), post.getFloat("raa", 0.1f), post.getFloat("rar", 0.1f));
- GraphPlotter.Ribbon rEdge = new GraphPlotter.Ribbon(post.getFloat("rel", 0.05f), post.getFloat("rea", 0.1f), post.getFloat("rer", 0.1f));
- int pa = post.getInt("pa", 0);
+ final GraphPlotter.Ribbon rAll = new GraphPlotter.Ribbon(post.getFloat("ral", 0.1f), post.getFloat("raa", 0.1f), post.getFloat("rar", 0.1f));
+ final GraphPlotter.Ribbon rEdge = new GraphPlotter.Ribbon(post.getFloat("rel", 0.05f), post.getFloat("rea", 0.1f), post.getFloat("rer", 0.1f));
+ final int pa = post.getInt("pa", 0);
for (int i = 0; i < pa; i++) graph = graph.physics(rAll, rEdge);
}
@@ -140,25 +142,25 @@ public class WebStructurePicture_p {
}
private static final int place(
- final GraphPlotter graph, final WebStructureGraph structure, String hostName,
- int bf, int maxnodes, final long timeout, int nextlayer, final int maxlayer, final int cyc) {
- Point pivotpoint = graph.getNode(hostName);
+ final GraphPlotter graph, final WebStructureGraph structure, final String hostName,
+ final int bf, int maxnodes, final long timeout, int nextlayer, final int maxlayer, final int cyc) {
+ final Point pivotpoint = graph.getNode(hostName);
int branches = 0;
if (nextlayer == maxlayer) return branches;
nextlayer++;
final double radius = 1.0 / (1 << nextlayer);
final Map<String, Integer> next = structure.outgoingReferencesByHostName(hostName);
- ClusteredScoreMap<String> next0 = new ClusteredScoreMap<String>(false);
- for (Map.Entry<String, Integer> entry: next.entrySet()) next0.set(entry.getKey(), entry.getValue());
+ final ClusteredScoreMap<String> next0 = new ClusteredScoreMap<String>(false);
+ for (final Map.Entry<String, Integer> entry: next.entrySet()) next0.set(entry.getKey(), entry.getValue());
// first set points to next hosts
final Set<String> targetHostNames = new HashSet<String>();
int maxtargetrefs = 8, maxthisrefs = 8;
int targetrefs, thisrefs;
double rr, re;
- Iterator<String> i = next0.keys(false);
+ final Iterator<String> i = next0.keys(false);
while (i.hasNext()) {
- String targethash = i.next();
- String targethost = structure.hostHash2hostName(targethash);
+ final String targethash = i.next();
+ final String targethost = structure.hostHash2hostName(targethash);
if (targethost == null) continue;
thisrefs = next.get(targethash).intValue();
targetrefs = structure.referencesCount(targethash); // can be cpu/time-critical
@@ -177,7 +179,7 @@ public class WebStructurePicture_p {
}
// recursively set next hosts
int nextnodes;
- for (String targetHostName: targetHostNames) {
+ for (final String targetHostName: targetHostNames) {
nextnodes = ((maxnodes <= 0) || (System.currentTimeMillis() >= timeout)) ? 0 : place(graph, structure, targetHostName, bf, maxnodes, timeout, nextlayer, maxlayer, cyc);
branches += nextnodes;
maxnodes -= nextnodes;
diff --git a/htroot/WikiHelp.java b/source/net/yacy/htroot/WikiHelp.java
similarity index 95%
rename from htroot/WikiHelp.java
rename to source/net/yacy/htroot/WikiHelp.java
index 334c99dea..f22e4afa0 100644
--- a/htroot/WikiHelp.java
+++ b/source/net/yacy/htroot/WikiHelp.java
@@ -27,6 +27,8 @@
// javac -classpath .:../classes WikiHelp.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
diff --git a/htroot/YaCySearchPluginFF.java b/source/net/yacy/htroot/YaCySearchPluginFF.java
similarity index 91%
rename from htroot/YaCySearchPluginFF.java
rename to source/net/yacy/htroot/YaCySearchPluginFF.java
index 378b235b1..01f2fd8bc 100644
--- a/htroot/YaCySearchPluginFF.java
+++ b/source/net/yacy/htroot/YaCySearchPluginFF.java
@@ -27,6 +27,8 @@
//javac -classpath .:../Classes Status.java
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
@@ -44,8 +46,8 @@ public class YaCySearchPluginFF {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
- String host = header.getServerName();
- int port = header.getServerPort();
+ final String host = header.getServerName();
+ final int port = header.getServerPort();
prop.put("host", host);
prop.put("port", port);
diff --git a/htroot/autoconfig.java b/source/net/yacy/htroot/autoconfig.java
similarity index 92%
rename from htroot/autoconfig.java
rename to source/net/yacy/htroot/autoconfig.java
index 3c3598f02..7487f5867 100644
--- a/htroot/autoconfig.java
+++ b/source/net/yacy/htroot/autoconfig.java
@@ -27,6 +27,8 @@
//javac -classpath .:../Classes Status.java
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
@@ -49,8 +51,8 @@ public class autoconfig {
final boolean yacyonly = env.getConfigBool(SwitchboardConstants.PROXY_YACY_ONLY, false);
- String host = header.getServerName();
- int port = header.getServerPort();
+ final String host = header.getServerName();
+ final int port = header.getServerPort();
prop.put("yacy", yacyonly ? "0" : "1");
prop.put("yacy_host", host);
diff --git a/htroot/compare_yacy.java b/source/net/yacy/htroot/compare_yacy.java
similarity index 98%
rename from htroot/compare_yacy.java
rename to source/net/yacy/htroot/compare_yacy.java
index 1a2b8d958..4689762e0 100644
--- a/htroot/compare_yacy.java
+++ b/source/net/yacy/htroot/compare_yacy.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.LinkedHashMap;
import java.util.Map;
@@ -82,7 +84,7 @@ public class compare_yacy {
prop.put("searchengines", searchengines.size());
int i = 0;
- for (String name: searchengines.keySet()) {
+ for (final String name: searchengines.keySet()) {
prop.putHTML("searchengines_" + i + "_searchengine", name);
prop.put("searchengines_" + i + "_leftengine", name.equals(default_left) ? 1 : 0);
prop.put("searchengines_" + i + "_rightengine", name.equals(default_right) ? 1 : 0);
diff --git a/htroot/cytag.java b/source/net/yacy/htroot/cytag.java
similarity index 94%
rename from htroot/cytag.java
rename to source/net/yacy/htroot/cytag.java
index 2a629fe2a..e7a11c80c 100644
--- a/htroot/cytag.java
+++ b/source/net/yacy/htroot/cytag.java
@@ -7,7 +7,7 @@
// $LastChangedBy$
//
// LICENSE
-//
+//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@@ -22,6 +22,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.awt.Image;
import java.io.File;
import java.io.IOException;
@@ -38,14 +40,14 @@ import net.yacy.server.serverSwitch;
public class cytag {
-
+
public static Image respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
-
+
final Switchboard sb = (Switchboard)env;
final MultiProtocolURL referer = header.referer();
-
+
// harvest request information
- StringBuilder connect = new StringBuilder();
+ final StringBuilder connect = new StringBuilder();
connect.append('{');
appendJSON(connect, "time", GenericFormatter.SHORT_MILSEC_FORMATTER.format());
appendJSON(connect, "trail", (referer == null) ? "" : referer.toNormalform(false));
@@ -56,11 +58,11 @@ public class cytag {
appendJSON(connect, "ip", (ip == null) ? "" : ip);
appendJSON(connect, "agent", header.get("User-Agent", ""));
connect.append('}');
-
+
if (sb.trail.size() >= 100) sb.trail.remove();
sb.trail.add(connect.toString());
//Log.logInfo("CYTAG", "catched trail - " + connect.toString());
-
+
final String defaultimage;
if (post != null && post.get("icon", "").equals("invisible")) {
defaultimage = "invisible.png";
@@ -68,7 +70,7 @@ public class cytag {
defaultimage = "redpillmini.png";
}
final File iconfile = new File(sb.getAppPath(), "/htroot/env/grafics/" + defaultimage);
-
+
byte[] imgb = null;
try {
imgb = FileUtils.read(iconfile);
@@ -76,13 +78,13 @@ public class cytag {
return null;
}
if (imgb == null) return null;
-
+
// read image
final Image image = ImageParser.parse("cytag.png", imgb);
return image;
}
-
+
private static final void appendJSON(final StringBuilder sb, final String key, final String value) {
if (sb.length() > 2) sb.append(',');
sb.append('\"');
diff --git a/htroot/goto_p.java b/source/net/yacy/htroot/goto_p.java
similarity index 93%
rename from htroot/goto_p.java
rename to source/net/yacy/htroot/goto_p.java
index 24ccb1d9c..865f04d90 100644
--- a/htroot/goto_p.java
+++ b/source/net/yacy/htroot/goto_p.java
@@ -26,6 +26,8 @@
// javac -classpath .:../Classes goto_p.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.util.Set;
import net.yacy.cora.protocol.RequestHeader;
@@ -42,7 +44,7 @@ public class goto_p {
/**
* url parameter
*
- * hash= of remote peer
+ * hash= of remote peer
* path= path part to forward to
*/
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
@@ -55,7 +57,7 @@ public class goto_p {
}
if (hash != null) {
- Seed seed = sb.peers.getConnected(hash);
+ final Seed seed = sb.peers.getConnected(hash);
if (seed != null) {
final Set<String> ips = seed.getIPs();
diff --git a/htroot/imagetest.java b/source/net/yacy/htroot/imagetest.java
similarity index 97%
rename from htroot/imagetest.java
rename to source/net/yacy/htroot/imagetest.java
index 1cd624645..89f82c75e 100644
--- a/htroot/imagetest.java
+++ b/source/net/yacy/htroot/imagetest.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
diff --git a/htroot/index.java b/source/net/yacy/htroot/index.java
similarity index 95%
rename from htroot/index.java
rename to source/net/yacy/htroot/index.java
index ce36e34f6..61e4acd97 100644
--- a/htroot/index.java
+++ b/source/net/yacy/htroot/index.java
@@ -28,6 +28,8 @@
// javac -classpath .:../classes index.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import net.yacy.cora.document.analysis.Classification;
import net.yacy.cora.document.analysis.Classification.ContentDomain;
import net.yacy.cora.protocol.RequestHeader;
@@ -55,7 +57,7 @@ public class index {
// access control
String authenticatedUserName = null;
final boolean adminAuthenticated = sb.verifyAuthentication(header);
-
+
if(adminAuthenticated) {
authenticatedUserName = sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin");
} else {
@@ -64,7 +66,7 @@ public class index {
authenticatedUserName = user.getUserName();
}
}
- boolean authenticated = adminAuthenticated || authenticatedUserName != null;
+ final boolean authenticated = adminAuthenticated || authenticatedUserName != null;
if (post != null) {
if (post.containsKey("publicPage") && !adminAuthenticated) { // Old style parameter : still in use ?
prop.authenticationRequired();
@@ -75,16 +77,16 @@ public class index {
return prop;
}
}
-
+
prop.put("authSearch", authenticated);
-
+
boolean global = (post == null) ? true : post.get("resource", "global").equals("global");
final boolean focus = (post == null) ? true : post.get("focus", "1").equals("1");
int searchoptions = (post == null) ? 0 : Math.min(1, post.getInt("searchoptions", 0));
if (!sb.getConfigBool("search.options", true)) {
searchoptions = 0;
- } else {
+ } else {
// show heuristic hint on search option screen (only if heuristic is ON by config)
prop.put("searchoptions_heuristic", sb.getConfigBool(SwitchboardConstants.HEURISTIC_OPENSEARCH, false));
// show date search options (only if dates_in_content_dts search target field is active)
@@ -158,9 +160,9 @@ public class index {
prop.put("searchdomswitches_searchapp_check", (contentdom == ContentDomain.APP) ? "1" : "0");
prop.put("search.navigation", sb.getConfig("search.navigation", "all") );
prop.put("search.verify", sb.getConfig("search.verify", "iffresh") );
-
+
handleTopNavBarLoginSection(header, sb, prop, authenticatedUserName);
-
+
// online caution timing
sb.localSearchLastAccess = System.currentTimeMillis();
@@ -186,15 +188,15 @@ public class index {
} else {
/* Show a login link */
prop.put("showLogin", 2);
-
+
/* The login link targets the same URL as the current location, just adding the 'auth' parameter to indicates that access to extended search features is desired */
- StringBuilder loginURL = new StringBuilder("index.html?auth");
+ final StringBuilder loginURL = new StringBuilder("index.html?auth");
final String query = header.getQueryString();
if(query != null) {
loginURL.append("&").append(query);
}
-
-
+
+
prop.put("showLogin_loginURL", loginURL.toString());
}
} else {
diff --git a/htroot/jslicense.java b/source/net/yacy/htroot/jslicense.java
similarity index 95%
rename from htroot/jslicense.java
rename to source/net/yacy/htroot/jslicense.java
index 27c1e3c54..45d3115a0 100755
--- a/htroot/jslicense.java
+++ b/source/net/yacy/htroot/jslicense.java
@@ -27,6 +27,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
diff --git a/htroot/mediawiki_p.java b/source/net/yacy/htroot/mediawiki_p.java
similarity index 99%
rename from htroot/mediawiki_p.java
rename to source/net/yacy/htroot/mediawiki_p.java
index 51728ef30..69b9fb0db 100644
--- a/htroot/mediawiki_p.java
+++ b/source/net/yacy/htroot/mediawiki_p.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
diff --git a/htroot/opensearchdescription.java b/source/net/yacy/htroot/opensearchdescription.java
similarity index 98%
rename from htroot/opensearchdescription.java
rename to source/net/yacy/htroot/opensearchdescription.java
index 8364fea59..03fd790c1 100644
--- a/htroot/opensearchdescription.java
+++ b/source/net/yacy/htroot/opensearchdescription.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.http.servlets.YaCyDefaultServlet;
import net.yacy.search.Switchboard;
@@ -40,7 +42,7 @@ public class opensearchdescription {
if (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
final String thisContext = YaCyDefaultServlet.getContext(header, sb);
-
+
final serverObjects prop = new serverObjects();
prop.put("compareyacy", post != null && post.getBoolean("compare_yacy") ? 1 : 0);
prop.putXML("thisContext", thisContext);
diff --git a/htroot/osm.java b/source/net/yacy/htroot/osm.java
similarity index 94%
rename from htroot/osm.java
rename to source/net/yacy/htroot/osm.java
index 09828d3bb..1671e8957 100644
--- a/htroot/osm.java
+++ b/source/net/yacy/htroot/osm.java
@@ -7,17 +7,19 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
- *
+ *
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
- *
+ *
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.peers.graphics.EncodedImage;
@@ -47,13 +49,13 @@ public class osm {
}
final OSMTile.tileCoordinates coord = new OSMTile.tileCoordinates(lat, lon, zoom);
- RasterPlotter map = OSMTile.getCombinedTiles(coord, width, height);
+ final RasterPlotter map = OSMTile.getCombinedTiles(coord, width, height);
map.setDrawMode(DrawMode.MODE_SUB);
map.setColor(0xffffff);
/*
* copyright notice on OSM Tiles
* According to http://www.openstreetmap.org/copyright/ the (C) of the map tiles is (CC BY-SA)
- * while the OpenStreetMap raw data is licensed with (ODbL) http://opendatacommons.org/licenses/odbl/
+ * while the OpenStreetMap raw data is licensed with (ODbL) http://opendatacommons.org/licenses/odbl/
* Map tiles shall be underlined with the statement "(C) OpenStreetMap contributors". In our 5-dot character
* set the lowercase letters do not look good, so we use uppercase only.
* The (C) symbol is not available in our font, so we use the letters (C) instead.
diff --git a/htroot/rct_p.java b/source/net/yacy/htroot/rct_p.java
similarity index 99%
rename from htroot/rct_p.java
rename to source/net/yacy/htroot/rct_p.java
index a27e3e27a..7c7bf5384 100644
--- a/htroot/rct_p.java
+++ b/source/net/yacy/htroot/rct_p.java
@@ -25,6 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.net.MalformedURLException;
import java.util.Date;
import java.util.Iterator;
diff --git a/htroot/robots.java b/source/net/yacy/htroot/robots.java
similarity index 99%
rename from htroot/robots.java
rename to source/net/yacy/htroot/robots.java
index df87cf5e7..06b82ffd3 100644
--- a/htroot/robots.java
+++ b/source/net/yacy/htroot/robots.java
@@ -4,6 +4,8 @@
//$LastChangedBy$
//
+package net.yacy.htroot;
+
import java.io.File;
import java.util.ArrayList;
import java.util.List;
@@ -58,7 +60,7 @@ public class robots {
}
}
}
-
+
if (rbc.isLockedDisallowed()) {
prop.put(RobotsTxtConfig.ALL + "_" + RobotsTxtConfig.LOCKED, htrootFiles.size());
for (int i = 0; i < htrootFiles.size(); i++) {
diff --git a/htroot/sharedBlacklist_p.java b/source/net/yacy/htroot/sharedBlacklist_p.java
similarity index 96%
rename from htroot/sharedBlacklist_p.java
rename to source/net/yacy/htroot/sharedBlacklist_p.java
index c22059c12..18eb528b0 100644
--- a/htroot/sharedBlacklist_p.java
+++ b/source/net/yacy/htroot/sharedBlacklist_p.java
@@ -29,6 +29,8 @@
//javac -classpath .:../Classes Blacklist_p.java
//if the shell's current path is HTROOT
+package net.yacy.htroot;
+
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
@@ -53,8 +55,8 @@ import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.Seed;
import net.yacy.repository.Blacklist;
-import net.yacy.repository.BlacklistHostAndPath;
import net.yacy.repository.Blacklist.BlacklistType;
+import net.yacy.repository.BlacklistHostAndPath;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.SearchEventCache;
@@ -131,7 +133,7 @@ public class sharedBlacklist_p {
Iterator<String> otherBlacklist = null;
ListAccumulator otherBlacklists = null;
- ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
+ final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
if (post.containsKey("hash")) {
/* ======================================================
@@ -152,11 +154,11 @@ public class sharedBlacklist_p {
final String Port = seed.get(Seed.PORT, "8090");
final String peerName = seed.get(Seed.NAME, "<" + IP + ":" + Port + ">");
prop.putHTML("page_source", peerName);
-
+
downloadURLOld = seed.getPublicURL(IP,
sb.getConfigBool(SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED,
SwitchboardConstants.REMOTESEARCH_HTTPS_PREFERRED_DEFAULT)) + "/yacy/list.html?col=black";
-
+
// download the blacklist
try {
// get List
diff --git a/htroot/ssitestservlet.java b/source/net/yacy/htroot/ssitestservlet.java
similarity index 98%
rename from htroot/ssitestservlet.java
rename to source/net/yacy/htroot/ssitestservlet.java
index 274c57b41..fef629f25 100644
--- a/htroot/ssitestservlet.java
+++ b/source/net/yacy/htroot/ssitestservlet.java
@@ -25,6 +25,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
diff --git a/htroot/suggest.java b/source/net/yacy/htroot/suggest.java
similarity index 96%
rename from htroot/suggest.java
rename to source/net/yacy/htroot/suggest.java
index 609a93ff0..c82c0e1ce 100644
--- a/htroot/suggest.java
+++ b/source/net/yacy/htroot/suggest.java
@@ -22,6 +22,8 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Collection;
import java.util.ConcurrentModificationException;
@@ -59,7 +61,7 @@ public class suggest {
final String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, "");
final boolean json = ext.equals("json");
final boolean xml = ext.equals("xml");
-
+
// get query
final String originalquerystring = (post == null) ? "" : post.get("query", post.get("q", ""));
final String querystring = originalquerystring.trim();
@@ -71,10 +73,10 @@ public class suggest {
final Collection<StringBuilder> suggestions = didYouMean.getSuggestions(timeout, count, sb.index.fulltext().collectionSize() < 2000000);
//[#[query]#,[#{suggestions}##[text]##(eol)#,::#(/eol)##{/suggestions}#]]
synchronized (suggestions) {
- for (StringBuilder suggestion: suggestions) {
+ for (final StringBuilder suggestion: suggestions) {
if (c >= meanMax) break;
try {
- String s = suggestion.toString();
+ final String s = suggestion.toString();
if (json) {
prop.putJSON("suggestions_" + c + "_text", s);
} else if (xml) {
diff --git a/htroot/test.java b/source/net/yacy/htroot/test.java
similarity index 96%
rename from htroot/test.java
rename to source/net/yacy/htroot/test.java
index baa3efb01..70601bbb5 100644
--- a/htroot/test.java
+++ b/source/net/yacy/htroot/test.java
@@ -4,6 +4,8 @@
//$LastChangedBy$
//
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
diff --git a/htroot/yacyinteractive.java b/source/net/yacy/htroot/yacyinteractive.java
similarity index 94%
rename from htroot/yacyinteractive.java
rename to source/net/yacy/htroot/yacyinteractive.java
index 9a91763b9..644cfbfa0 100644
--- a/htroot/yacyinteractive.java
+++ b/source/net/yacy/htroot/yacyinteractive.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@@ -36,8 +38,8 @@ public class yacyinteractive {
private static long indeSizeCache = 0;
private static long indexSizeTime = 0;
-
- public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, serverObjects post, final serverSwitch env) {
+
+ public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
@@ -59,7 +61,7 @@ public class yacyinteractive {
prop.put("serverlist", query.isEmpty() ? 1 : 0);
prop.put("focus", focus ? 1 : 0);
- long t = System.currentTimeMillis();
+ final long t = System.currentTimeMillis();
if (t - indexSizeTime > 60000) {
indeSizeCache = sb.index.fulltext().collectionSize();
indexSizeTime = t;
diff --git a/htroot/yacysearch.java b/source/net/yacy/htroot/yacysearch.java
similarity index 97%
rename from htroot/yacysearch.java
rename to source/net/yacy/htroot/yacysearch.java
index c318f7baf..ac728a4a3 100644
--- a/htroot/yacysearch.java
+++ b/source/net/yacy/htroot/yacysearch.java
@@ -27,6 +27,10 @@
// javac -classpath .:../classes yacysearch.java
// if the shell's current path is HTROOT
+package net.yacy.htroot;
+
+import static net.yacy.repository.BlacklistHelper.addBlacklistEntry;
+
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.IDN;
@@ -80,7 +84,6 @@ import net.yacy.peers.EventChannel;
import net.yacy.peers.NewsPool;
import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.repository.Blacklist;
-import static net.yacy.repository.BlacklistHelper.addBlacklistEntry;
import net.yacy.search.EventTracker;
import net.yacy.search.SearchAccessRateConstants;
import net.yacy.search.Switchboard;
@@ -149,16 +152,16 @@ public class yacysearch {
prop.put("authSearch", authenticatedUserName != null);
// produce vocabulary navigation sidebars
- Collection<Tagging> vocabularies = LibraryProvider.autotagging.getVocabularies();
+ final Collection<Tagging> vocabularies = LibraryProvider.autotagging.getVocabularies();
int j = 0;
- for (Tagging v: vocabularies) {
+ for (final Tagging v: vocabularies) {
prop.put("sidebarVocabulary_" + j + "_vocabulary", v.getName());
j++;
}
prop.put("sidebarVocabulary", j);
// get segment
- Segment indexSegment = sb.index;
+ final Segment indexSegment = sb.index;
final String EXT = header.get(HeaderFramework.CONNECTION_PROP_EXT, "");
final boolean rss = "rss.atom".contains(EXT);
@@ -166,15 +169,15 @@ public class yacysearch {
prop.put("promoteSearchPageGreeting", promoteSearchPageGreeting);
// adding some additional properties needed for the rss feed
- String peerContext = YaCyDefaultServlet.getContext(header, sb);
+ final String peerContext = YaCyDefaultServlet.getContext(header, sb);
prop.put("searchBaseURL", peerContext + "/yacysearch.html");
prop.put("rssYacyImageURL", peerContext + "/env/grafics/yacy.png");
prop.put("thisaddress", peerContext);
final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER);
final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, true) || clustersearch;
- boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;
+ final boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;
boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode);
- boolean stealthmode = p2pmode && !global;
+ final boolean stealthmode = p2pmode && !global;
if ( post == null || indexSegment == null || env == null || !searchAllowed ) {
if (indexSegment == null) ConcurrentLog.info("yacysearch", "indexSegment == null");
@@ -247,7 +250,7 @@ public class yacysearch {
}
// time zone
- int timezoneOffset = post.getInt("timezoneOffset", 0);
+ final int timezoneOffset = post.getInt("timezoneOffset", 0);
// collect search attributes
@@ -479,7 +482,7 @@ public class yacysearch {
// read collection
modifier.collection = post.get("collection", modifier.collection); // post arguments may overrule parsed collection values
- int stp = querystring.indexOf('*');
+ final int stp = querystring.indexOf('*');
if (stp >= 0) {
// if the star appears as a single entry, use the catchallstring
if (querystring.length() == 1) {
@@ -536,10 +539,10 @@ public class yacysearch {
}
int voc = 0;
- Collection<Tagging.Metatag> metatags = new ArrayList<Tagging.Metatag>(1);
+ final Collection<Tagging.Metatag> metatags = new ArrayList<Tagging.Metatag>(1);
while ((voc = querystring.indexOf("/vocabulary/", 0)) >= 0) {
String vocabulary = "";
- int ve = querystring.indexOf(' ', voc + 12);
+ final int ve = querystring.indexOf(' ', voc + 12);
if (ve < 0) {
vocabulary = querystring.substring(voc);
querystring = querystring.substring(0, voc).trim();
@@ -549,11 +552,11 @@ public class yacysearch {
}
modifier.add(vocabulary);
vocabulary = vocabulary.substring(12);
- int p = vocabulary.indexOf('/');
+ final int p = vocabulary.indexOf('/');
if (p > 0) {
- String k = vocabulary.substring(0, p);
- String v = vocabulary.substring(p + 1);
- Metatag mt = LibraryProvider.autotagging.metatag(k, v);
+ final String k = vocabulary.substring(0, p);
+ final String v = vocabulary.substring(p + 1);
+ final Metatag mt = LibraryProvider.autotagging.metatag(k, v);
if (mt != null) {
metatags.add(mt);
} else {
@@ -565,7 +568,7 @@ public class yacysearch {
int radius = 0;
double lon = 0.0d, lat = 0.0d, rad = 0.0d;
if ((radius = querystring.indexOf("/radius/")) >= 0) {
- int ve = querystring.indexOf(' ', radius + 8);
+ final int ve = querystring.indexOf(' ', radius + 8);
String geo = "";
if (ve < 0) {
geo = querystring.substring(radius);
@@ -575,7 +578,7 @@ public class yacysearch {
querystring = querystring.substring(0, radius) + querystring.substring(ve);
}
geo = geo.substring(8);
- String[] sp = geo.split("/");
+ final String[] sp = geo.split("/");
if (sp.length == 3) try {
lat = Double.parseDouble(sp[0]);
lon = Double.parseDouble(sp[1]);
@@ -717,7 +720,7 @@ public class yacysearch {
bmk.setProperty(Bookmark.BOOKMARK_QUERY, querystring);
bmk.addTag("/search"); // add to bookmark folder
bmk.addTag("searchresult"); // add tag
- String urlhash = post.get("bookmarkref");
+ final String urlhash = post.get("bookmarkref");
final URIMetadataNode urlentry = indexSegment.fulltext().getMetadata(UTF8.getBytes(urlhash));
if (urlentry != null && !urlentry.dc_title().isEmpty()) {
bmk.setProperty(Bookmark.BOOKMARK_TITLE, urlentry.dc_title());
@@ -735,13 +738,13 @@ public class yacysearch {
return prop;
}
- String blacklisturl = crypt.simpleDecode(post.get("blacklisturl", "")); // url
+ final String blacklisturl = crypt.simpleDecode(post.get("blacklisturl", "")); // url
try {
- MultiProtocolURL mpurl = new MultiProtocolURL(blacklisturl);
+ final MultiProtocolURL mpurl = new MultiProtocolURL(blacklisturl);
addBlacklistEntry(
Blacklist.defaultBlacklist(sb.listsPath),
mpurl.getHost() + "/.*");
- } catch (MalformedURLException e) {
+ } catch (final MalformedURLException e) {
}
}
@@ -1037,11 +1040,11 @@ public class yacysearch {
prop.put("focus", focus ? 1 : 0); // focus search field
prop.put("searchagain", global ? "1" : "0");
- String former = originalquerystring.replaceAll(Segment.catchallString, "*"); // hide catchallString in output
+ final String former = originalquerystring.replaceAll(Segment.catchallString, "*"); // hide catchallString in output
prop.putHTML("former", former);
try {
prop.put("formerEncoded", URLEncoder.encode(former, StandardCharsets.UTF_8.name()));
- } catch (UnsupportedEncodingException e) {
+ } catch (final UnsupportedEncodingException e) {
ConcurrentLog.warn("LOCAL_SEARCH", "Unsupported UTF-8 encoding!");
prop.put("formerEncoded", former);
}
diff --git a/htroot/yacysearch_location.java b/source/net/yacy/htroot/yacysearch_location.java
similarity index 97%
rename from htroot/yacysearch_location.java
rename to source/net/yacy/htroot/yacysearch_location.java
index 865cca097..3c119986f 100644
--- a/htroot/yacysearch_location.java
+++ b/source/net/yacy/htroot/yacysearch_location.java
@@ -17,6 +17,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.util.Date;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
@@ -44,7 +46,7 @@ public class yacysearch_location {
final serverObjects prop = new serverObjects();
prop.put("kml", 0);
-
+
final String peerContext = YaCyDefaultServlet.getContext(header, sb);
if (header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("kml") ||
@@ -68,7 +70,7 @@ public class yacysearch_location {
final double lat = post.getDouble("lat", 0.0d);
final double radius = post.getDouble("r", 0.0d);
//i.e. http://localhost:8090/yacysearch_location.kml?query=berlin&maximumTime=2000&maximumRecords=100
-
+
int placemarkCounter = 0;
if (query.length() > 0 && search_query) {
final Set<GeoLocation> locations = LibraryProvider.geoLoc.find(query, true);
@@ -77,8 +79,8 @@ public class yacysearch_location {
}
for (final GeoLocation location: locations) {
// write for all locations a point to this message
- double lo = location.lon();
- double la = location.lat();
+ final double lo = location.lon();
+ final double la = location.lat();
if (lo == 0.0d && la == 0.0d) continue; // bad location
prop.put("kml_placemark_" + placemarkCounter + "_location", location.getName());
prop.put("kml_placemark_" + placemarkCounter + "_name", location.getName());
@@ -105,8 +107,8 @@ public class yacysearch_location {
RSSMessage message;
loop: while ((message = results.poll(maximumTime, TimeUnit.MILLISECONDS)) != RSSMessage.POISON) {
if (message == null) break loop;
- double lo = message.getLon();
- double la = message.getLat();
+ final double lo = message.getLon();
+ final double la = message.getLat();
if (lo == 0.0d && la == 0.0d) continue; // bad location
prop.put("kml_placemark_" + placemarkCounter + "_location", message.getTitle());
prop.put("kml_placemark_" + placemarkCounter + "_name", message.getTitle());
diff --git a/htroot/yacysearchitem.java b/source/net/yacy/htroot/yacysearchitem.java
similarity index 94%
rename from htroot/yacysearchitem.java
rename to source/net/yacy/htroot/yacysearchitem.java
index a83d478f9..fafa3959d 100644
--- a/htroot/yacysearchitem.java
+++ b/source/net/yacy/htroot/yacysearchitem.java
@@ -24,6 +24,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.awt.Dimension;
import java.io.File;
import java.io.UnsupportedEncodingException;
@@ -65,7 +67,6 @@ import net.yacy.kelondro.util.Formatter;
import net.yacy.peers.NewsPool;
import net.yacy.peers.Seed;
import net.yacy.peers.graphics.ProfilingGraph;
-import net.yacy.repository.Blacklist;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@@ -160,7 +161,7 @@ public class yacysearchitem {
final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "");
final boolean noreferrer = sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_NOREFERRER, SwitchboardConstants.SEARCH_RESULT_NOREFERRER_DEFAULT);
- long timeout = item == 0 ? 10000 : (theSearch.query.isLocal() ? 1000 : 3000);
+ final long timeout = item == 0 ? 10000 : (theSearch.query.isLocal() ? 1000 : 3000);
if (theSearch.query.contentdom == Classification.ContentDomain.TEXT || theSearch.query.contentdom == Classification.ContentDomain.ALL) {
// text search
@@ -234,8 +235,8 @@ public class yacysearchitem {
// END interaction
- boolean isAtomFeed = header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("atom");
- String resultFileName = resultURL.getFileName();
+ final boolean isAtomFeed = header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("atom");
+ final String resultFileName = resultURL.getFileName();
prop.putHTML("content_target", target);
DigestURL faviconURL = null;
final boolean showFavicon = sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON,
@@ -264,7 +265,7 @@ public class yacysearchitem {
try {
prop.putXML("content_image_url", result.imageURL());
prop.put("content_image", 1);
- } catch (UnsupportedOperationException e) {
+ } catch (final UnsupportedOperationException e) {
/* May occur when the document embedded images information is incomplete to retrieve at least an valid image url*/
prop.put("content_image", 0);
@@ -273,12 +274,12 @@ public class yacysearchitem {
prop.put("content_urlhash", urlhash);
prop.put("content_ranking", Float.toString(result.score()));
- Date[] events = result.events();
- boolean showEvent = events != null && events.length > 0 && sb.getConfig("search.navigation", "").indexOf("date",0) >= 0;
+ final Date[] events = result.events();
+ final boolean showEvent = events != null && events.length > 0 && sb.getConfig("search.navigation", "").indexOf("date",0) >= 0;
prop.put("content_showEvent", showEvent ? 1 : 0);
- Collection<File> snapshotPaths = sb.getConfigBool("search.result.show.snapshots", true) ? Transactions.findPaths(result.url(), null, State.ANY) : null;
+ final Collection<File> snapshotPaths = sb.getConfigBool("search.result.show.snapshots", true) ? Transactions.findPaths(result.url(), null, State.ANY) : null;
if (fileType == FileType.HTML) { // html template specific settings
- boolean showKeywords = (sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS,
+ final boolean showKeywords = (sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS,
SwitchboardConstants.SEARCH_RESULT_SHOW_KEYWORDS_DEFAULT) && !result.dc_subject().isEmpty());
prop.put("content_showKeywords", showKeywords);
prop.put("content_showDate", sb.getConfigBool("search.result.show.date", true) && !showEvent ? 1 : 0);
@@ -298,8 +299,8 @@ public class yacysearchitem {
if (showKeywords) { // tokenize keywords
final StringTokenizer stoc = new StringTokenizer(result.dc_subject()," ");
String rawNavQueryModifier;
- Navigator navi = theSearch.navigatorPlugins.get("keywords");
- boolean naviAvail = navi != null;
+ final Navigator navi = theSearch.navigatorPlugins.get("keywords");
+ final boolean naviAvail = navi != null;
final int firstMaxKeywords = sb.getConfigInt(SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT,
SwitchboardConstants.SEARCH_RESULT_KEYWORDS_FISRT_MAX_COUNT_DEFAULT);
int i = 0;
@@ -347,9 +348,9 @@ public class yacysearchitem {
prop.put("content_showIndexBrowser_link", resultUrlstring);
if (sb.getConfigBool("search.result.show.vocabulary", true)) {
int c = 0;
- for (String key: result.getFieldNames()) {
+ for (final String key: result.getFieldNames()) {
if (key.startsWith("vocabulary_") && key.endsWith("_sxt")) {
- Collection<Object> terms = result.getFieldValues(key);
+ final Collection<Object> terms = result.getFieldValues(key);
prop.putHTML("content_showVocabulary_vocabulary_" + c + "_name", key.substring(11, key.length() - 4));
prop.putHTML("content_showVocabulary_vocabulary_" + c + "_terms", terms.toString());
c++;
@@ -447,8 +448,8 @@ public class yacysearchitem {
final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER);
final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, true) || clustersearch;
- boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;
- boolean stealthmode = p2pmode && theSearch.query.isLocal();
+ final boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;
+ final boolean stealthmode = p2pmode && theSearch.query.isLocal();
if ((sb.getConfigBool(SwitchboardConstants.HEURISTIC_SEARCHRESULTS, false) ||
(sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ACTIVE, false) && sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ENABLED, false) && Memory.getSystemLoadAverage() < 1.0)) &&
!stealthmode) sb.heuristicSearchResults(result);
@@ -631,16 +632,16 @@ public class yacysearchitem {
* @return favicon URL or null when even default favicon URL can not be generated
* @throws NullPointerException when one requested parameter is null
*/
- protected static DigestURL getFaviconURL(final URIMetadataNode result, Dimension preferredSize) {
+ protected static DigestURL getFaviconURL(final URIMetadataNode result, final Dimension preferredSize) {
/*
* We look preferably for a standard icon with preferred size, but
* accept as a fallback other icons below 128x128 or with no known size
*/
- IconEntry faviconEntry = result.getFavicon(preferredSize);
+ final IconEntry faviconEntry = result.getFavicon(preferredSize);
DigestURL faviconURL;
if (faviconEntry == null) {
try {
- String defaultFaviconURL = result.url().getProtocol() + "://" + result.url().getHost()
+ final String defaultFaviconURL = result.url().getProtocol() + "://" + result.url().getHost()
+ ((result.url().getPort() != -1) ? (":" + result.url().getPort()) : "") + "/favicon.ico";
faviconURL = new DigestURL(defaultFaviconURL);
} catch (final MalformedURLException e1) {
@@ -662,9 +663,9 @@ public class yacysearchitem {
* @return url to propose in search result or empty string when faviconURL
* is null
*/
- private static String processFaviconURL(final boolean hasFullViewingRights, DigestURL faviconURL) {
+ private static String processFaviconURL(final boolean hasFullViewingRights, final DigestURL faviconURL) {
/* Only use licence code for non authentified users. For authenticated users licence would never be released and would unnecessarily fill URLLicense.permissions. */
- StringBuilder contentFaviconURL = new StringBuilder();
+ final StringBuilder contentFaviconURL = new StringBuilder();
if (faviconURL != null) {
final String iconUrlExt = MultiProtocolURL.getFileExtension(faviconURL.getFileName());
/* Image format ouput for ViewFavicon servlet : default is png, except with gif and svg icons */
@@ -695,9 +696,9 @@ public class yacysearchitem {
*/
private static void addAuthorizedActions(final Switchboard sb, final serverObjects prop,
final SearchEvent theSearch, final String resultUrlstring, final String resource, final String origQ,
- final String urlhash, UserDB.Entry user) {
+ final String urlhash, final UserDB.Entry user) {
// check if url exists in bookmarks
- boolean bookmarkexists = sb.bookmarksDB.getBookmark(urlhash) != null;
+ final boolean bookmarkexists = sb.bookmarksDB.getBookmark(urlhash) != null;
if (user == null || user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT)) {
prop.put("content_authorized_bookmark", !bookmarkexists);
} else
@@ -715,17 +716,17 @@ public class yacysearchitem {
String encodedURLString;
try {
encodedURLString = URLEncoder.encode(crypt.simpleEncode(resultUrlstring), StandardCharsets.UTF_8.name());
- } catch (UnsupportedEncodingException e1) {
+ } catch (final UnsupportedEncodingException e1) {
ConcurrentLog.warn("YACY_SEARCH_ITEM", "UTF-8 encoding is not supported!");
encodedURLString = crypt.simpleEncode(resultUrlstring);
}
final String bookmarkLink = linkBuilder.append("&bookmarkurl=").append(encodedURLString).append("&bookmarkref=" + urlhash).toString();
linkBuilder.setLength(baseUrlLength);
- String deleteLink = linkBuilder.append("&deleteref=").append(urlhash).toString();
+ final String deleteLink = linkBuilder.append("&deleteref=").append(urlhash).toString();
linkBuilder.setLength(baseUrlLength);
- String recommendLink = linkBuilder.append("&recommendref=").append(urlhash).toString();
+ final String recommendLink = linkBuilder.append("&recommendref=").append(urlhash).toString();
linkBuilder.setLength(baseUrlLength);
final String blacklistLink = linkBuilder.append("&blacklisturl=").append(encodedURLString).toString();
@@ -760,10 +761,10 @@ public class yacysearchitem {
* @param noreferrer set to true when the noreferrer link type should be added to the original image source links
*/
private static void processImage(final Switchboard sb, final serverObjects prop, final int item,
- final SearchEvent theSearch, final String target_special_pattern, long timeout, boolean fullViewingRights, final boolean noreferrer) {
+ final SearchEvent theSearch, final String target_special_pattern, final long timeout, final boolean fullViewingRights, final boolean noreferrer) {
prop.put("content", theSearch.query.contentdom.getCode() + 1); // switch on specific content
try {
- SearchEvent.ImageResult image = theSearch.oneImageResult(item, timeout, theSearch.query.isStrictContentDom());
+ final SearchEvent.ImageResult image = theSearch.oneImageResult(item, timeout, theSearch.query.isStrictContentDom());
final String imageUrlstring = image.imageUrl.toNormalform(true);
final String imageUrlExt = MultiProtocolURL.getFileExtension(image.imageUrl.getFileName());
final String target = sb.getConfig(imageUrlstring.matches(target_special_pattern) ? SwitchboardConstants.SEARCH_TARGET_SPECIAL : SwitchboardConstants.SEARCH_TARGET_DEFAULT, "_self");
@@ -772,7 +773,7 @@ public class yacysearchitem {
/* Image format ouput for ViewImage servlet : default is png, except with gif and svg images */
final String viewImageExt = !imageUrlExt.isEmpty() && ImageViewer.isBrowserRendered(imageUrlExt) ? imageUrlExt : "png";
/* Thumb URL */
- StringBuilder thumbURLBuilder = new StringBuilder("ViewImage.").append(viewImageExt).append("?maxwidth=")
+ final StringBuilder thumbURLBuilder = new StringBuilder("ViewImage.").append(viewImageExt).append("?maxwidth=")
.append(DEFAULT_IMG_WIDTH).append("&maxheight=").append(DEFAULT_IMG_HEIGHT)
.append("&isStatic=true&quadratic");
/* Only use licence code for non authentified users. For authenticated users licence would never be released and would unnecessarily fill URLLicense.permissions. */
@@ -781,7 +782,7 @@ public class yacysearchitem {
} else {
thumbURLBuilder.append("&code=").append(URLLicense.aquireLicense(image.imageUrl));
}
- String thumbURL = thumbURLBuilder.toString();
+ final String thumbURL = thumbURLBuilder.toString();
prop.putHTML("content_item_hrefCache", thumbURL);
/* Full size preview URL */
if(fullViewingRights) {
@@ -807,8 +808,8 @@ public class yacysearchitem {
itemWidth = "";
itemHeight = "100%";
if(image.height > 0) {
- double scale = ((double)DEFAULT_IMG_HEIGHT) / ((double)image.height);
- int margin = (int)((image.height - image.width) * (scale / 2.0));
+ final double scale = ((double)DEFAULT_IMG_HEIGHT) / ((double)image.height);
+ final int margin = (int)((image.height - image.width) * (scale / 2.0));
itemStyle = "margin-left: " + margin + "px;";
}
} else {
@@ -816,8 +817,8 @@ public class yacysearchitem {
itemWidth = "100%";
itemHeight = "";
if(image.height > image.width && image.width > 0) {
- double scale = ((double)DEFAULT_IMG_WIDTH) / ((double)image.width);
- int margin = (int)((image.width - image.height) * (scale / 2.0));
+ final double scale = ((double)DEFAULT_IMG_WIDTH) / ((double)image.width);
+ final int margin = (int)((image.width - image.height) * (scale / 2.0));
itemStyle = "margin-top: " + margin + "px;";
}
}
@@ -833,7 +834,7 @@ public class yacysearchitem {
prop.put("content_item_sourcedom", image.sourceUrl.getHost());
prop.put("content_item_nl", (item == theSearch.query.offset) ? 0 : 1);
prop.put("content_item", 1);
- } catch (MalformedURLException e) {
+ } catch (final MalformedURLException e) {
prop.put("content_item", "0");
}
}
diff --git a/htroot/yacysearchlatestinfo.java b/source/net/yacy/htroot/yacysearchlatestinfo.java
similarity index 97%
rename from htroot/yacysearchlatestinfo.java
rename to source/net/yacy/htroot/yacysearchlatestinfo.java
index 321fe466c..5a77b0fd1 100644
--- a/htroot/yacysearchlatestinfo.java
+++ b/source/net/yacy/htroot/yacysearchlatestinfo.java
@@ -1,3 +1,5 @@
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.data.UserDB;
import net.yacy.http.servlets.TemplateMissingParameterException;
@@ -16,10 +18,10 @@ public class yacysearchlatestinfo {
if (post == null) {
throw new TemplateMissingParameterException("The eventID parameter is required");
}
-
+
final serverObjects prop = new serverObjects();
- Switchboard sb = (Switchboard) env;
-
+ final Switchboard sb = (Switchboard) env;
+
final boolean adminAuthenticated = sb.verifyAuthentication(header);
final UserDB.Entry user = sb.userDB != null ? sb.userDB.getUser(header) : null;
final boolean userAuthenticated = (user != null && user.hasRight(UserDB.AccessRight.EXTENDED_SEARCH_RIGHT));
diff --git a/htroot/yacysearchpagination.java b/source/net/yacy/htroot/yacysearchpagination.java
similarity index 97%
rename from htroot/yacysearchpagination.java
rename to source/net/yacy/htroot/yacysearchpagination.java
index 1a41b5f61..4556b216f 100644
--- a/htroot/yacysearchpagination.java
+++ b/source/net/yacy/htroot/yacysearchpagination.java
@@ -21,6 +21,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.http.servlets.TemplateMissingParameterException;
import net.yacy.search.Switchboard;
@@ -61,7 +63,7 @@ public class yacysearchpagination {
final int defaultItemsPerPage = sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10);
/* Detailed rules on items per page limits are handle in yacysearch.html */
- int itemsPerPage = Math.max(1, post.getInt("maximumRecords", defaultItemsPerPage));
+ final int itemsPerPage = Math.max(1, post.getInt("maximumRecords", defaultItemsPerPage));
final SearchEvent theSearch = SearchEventCache.getEvent(eventID);
if (theSearch == null) {
@@ -71,9 +73,9 @@ public class yacysearchpagination {
prop.put("pagination", false);
} else {
prop.put("pagination", true);
-
+
final RequestHeader.FileType fileType = header.fileType();
-
+
if(jsResort) {
/* Pagination links are processed on browser side : just prepare prev and next buttons */
prop.put("pagination_hidePagination", true);
@@ -125,7 +127,7 @@ public class yacysearchpagination {
.navurl(fileType, activePage + 1, theSearch.query, null, false, authFeatures).toString());
}
}
-
+
}
diff --git a/htroot/yacysearchtrailer.java b/source/net/yacy/htroot/yacysearchtrailer.java
similarity index 92%
rename from htroot/yacysearchtrailer.java
rename to source/net/yacy/htroot/yacysearchtrailer.java
index c32fdaecb..3b9e7a266 100644
--- a/htroot/yacysearchtrailer.java
+++ b/source/net/yacy/htroot/yacysearchtrailer.java
@@ -19,6 +19,8 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+package net.yacy.htroot;
+
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
@@ -81,7 +83,7 @@ public class yacysearchtrailer {
/*
* Authenticated search is explicitely requested here
* but no authentication is provided : ask now for authentication.
- * Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information
+ * Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information
* and as this page is not private, protected features would simply be hidden without asking browser again for authentication.
* (see mantis 766 : http://mantis.tokeek.de/view.php?id=766) *
*/
@@ -99,12 +101,12 @@ public class yacysearchtrailer {
final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER);
final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, true) || clustersearch;
- boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;
- boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode);
- boolean stealthmode = p2pmode && !global;
+ final boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;
+ final boolean global = post == null || (!post.get("resource-switch", post.get("resource", "global")).equals("local") && p2pmode);
+ final boolean stealthmode = p2pmode && !global;
// compose search navigation
- ContentDomain contentdom = theSearch.getQuery().contentdom;
+ final ContentDomain contentdom = theSearch.getQuery().contentdom;
prop.put("searchdomswitches",
sb.getConfigBool("search.text", true)
|| sb.getConfigBool("search.audio", true)
@@ -169,22 +171,22 @@ public class yacysearchtrailer {
navigatorIterator = topicNavigator.keys(false);
int i = 0;
// first sort the list to a form where the greatest element is in the middle
- LinkedList<Map.Entry<String, Integer>> cloud = new LinkedList<Map.Entry<String, Integer>>();
+ final LinkedList<Map.Entry<String, Integer>> cloud = new LinkedList<Map.Entry<String, Integer>>();
int mincount = Integer.MAX_VALUE; int maxcount = 0;
while (i < TOPWORDS_MAXCOUNT && navigatorIterator.hasNext()) {
name = navigatorIterator.next();
count = topicNavigator.get(name);
if (count == 0) break;
if (name == null) continue;
- int normcount = (count + TOPWORDS_MAXCOUNT - i) / 2;
+ final int normcount = (count + TOPWORDS_MAXCOUNT - i) / 2;
if (normcount > maxcount) maxcount = normcount;
if (normcount < mincount) mincount = normcount;
- Map.Entry<String, Integer> entry = new AbstractMap.SimpleEntry<String, Integer>(name, normcount);
+ final Map.Entry<String, Integer> entry = new AbstractMap.SimpleEntry<String, Integer>(name, normcount);
if (cloud.size() % 2 == 0) cloud.addFirst(entry); else cloud.addLast(entry); // alternating add entry to first or last position.
i++;
}
i= 0;
- for (Map.Entry<String, Integer> entry: cloud) {
+ for (final Map.Entry<String, Integer> entry: cloud) {
name = entry.getKey();
count = entry.getValue();
prop.put(fileType, "nav-topics_element_" + i + "_modifier", name);
@@ -211,12 +213,12 @@ public class yacysearchtrailer {
prop.put("nav-protocols", 1);
//int httpCount = theSearch.protocolNavigator.delete("http");
//int httpsCount = theSearch.protocolNavigator.delete("https");
- //theSearch.protocolNavigator.inc("http(s)", httpCount + httpsCount);
+ //theSearch.protocolNavigator.inc("http(s)", httpCount + httpsCount);
navigatorIterator = theSearch.protocolNavigator.keys(false);
int i = 0, pos = 0, neg = 0;
String nav, rawNav;
- String oldQuery = theSearch.query.getQueryGoal().query_original; // prepare hack to make radio-button like navigation
- String oldProtocolModifier = theSearch.query.modifier.protocol;
+ final String oldQuery = theSearch.query.getQueryGoal().query_original; // prepare hack to make radio-button like navigation
+ final String oldProtocolModifier = theSearch.query.modifier.protocol;
if (oldProtocolModifier != null && oldProtocolModifier.length() > 0) {theSearch.query.modifier.remove("/" + oldProtocolModifier); theSearch.query.modifier.remove(oldProtocolModifier);}
theSearch.query.modifier.protocol = "";
theSearch.query.getQueryGoal().query_original = oldQuery.replaceAll(" /https", "").replaceAll(" /http", "").replaceAll(" /ftp", "").replaceAll(" /smb", "").replaceAll(" /file", "");
@@ -235,7 +237,7 @@ public class yacysearchtrailer {
prop.put(fileType, "nav-protocols_element_" + i + "_modifier", nav);
url = QueryParams.navurl(fileType, 0, theSearch.query, rawNav, false, authenticated).toString();
} else {
- neg++;
+ neg++;
prop.put("nav-protocols_element_" + i + "_on", 1);
prop.put("nav-protocols_element_" + i + "_onclick", 1);
prop.put(fileType, "nav-protocols_element_" + i + "_modifier", "-" + nav);
@@ -280,15 +282,15 @@ public class yacysearchtrailer {
if(count == 0) {
continue;
}
- String shortname = name.substring(0, 10);
- long d = Instant.parse(name).toEpochMilli();
- Date dd = new Date(d);
+ final String shortname = name.substring(0, 10);
+ final long d = Instant.parse(name).toEpochMilli();
+ final Date dd = new Date(d);
if (fromconstraint != null && dd.before(fromconstraint)) continue;
if (toconstraint != null && dd.after(toconstraint)) break;
if (dx > 0) {
while (d - dx > AbstractFormatter.dayMillis && i < theSearch.getQuery().getDateFacetMaxCount()) {
dx += AbstractFormatter.dayMillis;
- String sn = new Date(dx).toInstant().toString().substring(0, 10);
+ final String sn = new Date(dx).toInstant().toString().substring(0, 10);
prop.put("nav-dates_element_" + i + "_on", 0);
prop.put(fileType, "nav-dates_element_" + i + "_name", sn);
prop.put("nav-dates_element_" + i + "_count", 0);
@@ -320,8 +322,8 @@ public class yacysearchtrailer {
final Map<String, ScoreMap<String>> vocabularyNavigators = theSearch.vocabularyNavigator;
if (vocabularyNavigators != null && !vocabularyNavigators.isEmpty()) {
int navvoccount = 0;
- vocnav: for (Map.Entry<String, ScoreMap<String>> ve: vocabularyNavigators.entrySet()) {
- String navname = ve.getKey();
+ vocnav: for (final Map.Entry<String, ScoreMap<String>> ve: vocabularyNavigators.entrySet()) {
+ final String navname = ve.getKey();
if (ve.getValue() == null || ve.getValue().isEmpty()) {
continue vocnav;
}
@@ -366,9 +368,9 @@ public class yacysearchtrailer {
// navigator plugins
int ni = 0;
- for (String naviname : theSearch.navigatorPlugins.keySet()) {
+ for (final String naviname : theSearch.navigatorPlugins.keySet()) {
- Navigator navi = theSearch.navigatorPlugins.get(naviname);
+ final Navigator navi = theSearch.navigatorPlugins.get(naviname);
if (navi.isEmpty()) {
continue;
}
@@ -383,16 +385,16 @@ public class yacysearchtrailer {
} else {
if(navi.getSort().getSortType() == NavigatorSortType.COUNT) {
if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) {
- navSort = 0;
+ navSort = 0;
} else {
navSort = 1;
}
} else {
if(navi.getSort().getSortDir() == NavigatorSortDirection.DESC) {
- navSort = 2;
+ navSort = 2;
} else {
navSort = 3;
- }
+ }
}
}
prop.put("navs_" + ni + "_navSort", navSort);
@@ -411,10 +413,10 @@ public class yacysearchtrailer {
rawNav = navi.getQueryModifier(name);
try {
nav = URLEncoder.encode(rawNav, StandardCharsets.UTF_8.name());
- } catch (UnsupportedEncodingException ex) {
+ } catch (final UnsupportedEncodingException ex) {
nav = "";
}
- boolean isactive = navi.modifieractive(theSearch.query.modifier, name);
+ final boolean isactive = navi.modifieractive(theSearch.query.modifier, name);
final String navUrl;
if (!isactive) {
pos++;
@@ -491,8 +493,8 @@ public class yacysearchtrailer {
* @param startRecord the zero based index of the first record to return
* @param meanMax the maximum number of suggestions ("Did you mean") to propose
*/
- private static void appendSearchFormValues(final String templatePrefix, final serverObjects post, final serverObjects prop,
- boolean global, final SearchEvent theSearch, final String former, final String snippetFetchStrategyName,
+ private static void appendSearchFormValues(final String templatePrefix, final serverObjects post, final serverObjects prop,
+ final boolean global, final SearchEvent theSearch, final String former, final String snippetFetchStrategyName,
final int startRecord, final int meanMax) {
prop.putHTML(templatePrefix + "former", former);
prop.put(templatePrefix + "authSearch", post.containsKey("auth"));