Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Upgraded the OpenGeoDB dump URL
luccioman8a29551c5405
diff --git a/htroot/DictionaryLoader_p.html b/htroot/DictionaryLoader_p.htmlindex a37d8647f..2c504886a 100644--- a/htroot/DictionaryLoader_p.html+++ b/htroot/DictionaryLoader_p.html@@ -130,24 +130,41 @@<p>With this file it is possible to find locations in Germany using the location (city) name, a zip code, a car sign or a telephone pre-dial number.</p><dl>- <dt><label>Download from</label></dt>- <dd>#[geo1URL]#</dd>+ #(geo0Status)#::+ <dt><label>Downloaded from</label></dt>+ <dd>#[URL]#</dd>+ <dt><label>Storage location</label></dt>+ <dd>#[Storage]#</dd>+ <dt><label>Status</label></dt>+ <dd><div class="commit">loaded - can be upgraded using the Load button for the new URL</div></dd>+ <dt>Action</dt>+ <dd>+ <input type="submit" name="geo0Deactivate" class="btn btn-sm btn-primary" value="Deactivate" />+ <input type="submit" name="geo0Remove" class="btn btn-sm btn-danger" value="Remove" />+ </dd>::+ #(/geo0Status)#+ #(geo0ActionRemoved)#::+ <dt>Result</dt><dd><div class="commit">deactivated and removed dictionary file</div></dd>::+ <dt>Result</dt><dd><div class="error">cannot remove dictionary file: #[error]#</div></dd>+ #(/geo0ActionRemoved)#+ #(geo0ActionDeactivated)#::+ <dt>Result</dt><dd><div class="commit">deactivated dictionary file</div></dd>::+ <dt>Result</dt><dd><div class="error">cannot de-activate dictionary file: #[error]#</div></dd>+ #(/geo0ActionDeactivated)#++ #(geo1Status)#::+ <dt><label>Downloaded from</label></dt>+ <dd>#[URL]#</dd><dt><label>Storage location</label></dt>- <dd>#[geo1Storage]#</dd>+ <dd>#[Storage]#</dd><dt><label>Status</label></dt>- <dd>#(geo1Status)#<div class="info">not loaded</div>::<div class="commit">loaded</div>::deactivated#(/geo1Status)#</dd>+ <dd><div class="commit">loaded - can be upgraded using the Load button for the new URL</div></dd><dt>Action</dt>- <dd>#(geo1Status)#- <input type="submit" name="geo1Load" class="btn btn-sm btn-primary" value="Load" />::+ <dd><input type="submit" name="geo1Deactivate" class="btn btn-sm btn-primary" value="Deactivate" />- <input type="submit" name="geo1Remove" class="btn btn-sm btn-danger" value="Remove" />::- <input type="submit" name="geo1Activate" class="btn btn-sm btn-primary" value="Activate" /><input type="submit" name="geo1Remove" class="btn btn-sm btn-danger" value="Remove" />- #(/geo1Status)#</dd>- #(geo1ActionLoaded)#::- <dt>Result</dt><dd><div class="commit">loaded and activated dictionary file</div></dd>::- <dt>Result</dt><dd><div class="error">loading of dictionary file failed: #[error]#</div></dd>- #(/geo1ActionLoaded)#+ </dd>::+ #(/geo1Status)##(geo1ActionRemoved)#::<dt>Result</dt><dd><div class="commit">deactivated and removed dictionary file</div></dd>::<dt>Result</dt><dd><div class="error">cannot remove dictionary file: #[error]#</div></dd>@@ -156,10 +173,38 @@<dt>Result</dt><dd><div class="commit">deactivated dictionary file</div></dd>::<dt>Result</dt><dd><div class="error">cannot de-activate dictionary file: #[error]#</div></dd>#(/geo1ActionDeactivated)#- #(geo1ActionActivated)#::++ <dt><label>Download from</label></dt>+ <dd>#[geo2URL]#</dd>+ <dt><label>Storage location</label></dt>+ <dd>#[geo2Storage]#</dd>+ <dt><label>Status</label></dt>+ <dd>#(geo2Status)#<div class="info">not loaded</div>::<div class="commit">loaded</div>::deactivated#(/geo2Status)#</dd>+ <dt>Action</dt>+ <dd>#(geo2Status)#+ <input type="submit" name="geo2Load" class="btn btn-sm btn-primary" value="Load" />::+ <input type="submit" name="geo2Deactivate" class="btn btn-sm btn-primary" value="Deactivate" />+ <input type="submit" name="geo2Remove" class="btn btn-sm btn-danger" value="Remove" />::+ <input type="submit" name="geo2Activate" class="btn btn-sm btn-primary" value="Activate" />+ <input type="submit" name="geo2Remove" class="btn btn-sm btn-danger" value="Remove" />+ #(/geo2Status)#</dd>+ #(geo2ActionLoaded)#::+ <dt>Result</dt><dd><div class="commit">loaded and activated dictionary file</div></dd>::+ <dt>Result</dt><dd><div class="error">loading of dictionary file failed: #[error]#</div></dd>::+ <dt>Result</dt><dd><div class="commit">loaded and upgraded dictionary file</div></dd>+ #(/geo2ActionLoaded)#+ #(geo2ActionRemoved)#::+ <dt>Result</dt><dd><div class="commit">deactivated and removed dictionary file</div></dd>::+ <dt>Result</dt><dd><div class="error">cannot remove dictionary file: #[error]#</div></dd>+ #(/geo2ActionRemoved)#+ #(geo2ActionDeactivated)#::+ <dt>Result</dt><dd><div class="commit">deactivated dictionary file</div></dd>::+ <dt>Result</dt><dd><div class="error">cannot de-activate dictionary file: #[error]#</div></dd>+ #(/geo2ActionDeactivated)#+ #(geo2ActionActivated)#::<dt>Result</dt><dd><div class="commit">activated dictionary file</div></dd>::<dt>Result</dt><dd><div class="error">cannot activate dictionary file: #[error]#</div></dd>- #(/geo1ActionActivated)#+ #(/geo2ActionActivated)#</dl></fieldset></form>diff --git a/htroot/DictionaryLoader_p.java b/htroot/DictionaryLoader_p.javaindex 3d1b3bfb8..09098a7df 100644--- a/htroot/DictionaryLoader_p.java+++ b/htroot/DictionaryLoader_p.java@@ -60,7 +60,9 @@ public class DictionaryLoader_p {for (final LibraryProvider.Dictionary dictionary: LibraryProvider.Dictionary.values()) {prop.put(dictionary.nickname + "URL", dictionary.url);+ prop.put(dictionary.nickname + "Status_URL", dictionary.url);prop.put(dictionary.nickname + "Storage", dictionary.file().toString());+ prop.put(dictionary.nickname + "Status_Storage", dictionary.file().toString());prop.put(dictionary.nickname + "Status", dictionary.file().exists() ? 1 : dictionary.fileDisabled().exists() ? 2 : 0);prop.put(dictionary.nickname + "ActionLoaded", 0);prop.put(dictionary.nickname + "ActionRemoved", 0);@@ -208,31 +210,56 @@ public class DictionaryLoader_p {prop.put("geon2ActionActivated", 1);}- // GEO1- if (post.containsKey("geo1Load")) {+ // GEO2+ if (post.containsKey("geo2Load")) {// load from the nettry {- final Response response = sb.loader.load(sb.loader.request(new DigestURL(LibraryProvider.Dictionary.GEODB1.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent);+ final Response response = sb.loader.load(sb.loader.request(new DigestURL(LibraryProvider.Dictionary.GEODB2.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent);final byte[] b = response.getContent();- FileUtils.copy(b, LibraryProvider.Dictionary.GEODB1.file());- LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB1.nickname);- LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null));+ FileUtils.copy(b, LibraryProvider.Dictionary.GEODB2.file());+ LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB2.nickname);+ boolean isUpgrade = false;+ if(LibraryProvider.Dictionary.GEODB0.file().exists()) {+ isUpgrade = true;+ FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.file());+ FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.fileDisabled());+ LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB0.nickname);+ } else if(LibraryProvider.Dictionary.GEODB1.file().exists()) {+ isUpgrade = true;+ FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.file());+ FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.fileDisabled());+ LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB1.nickname);+ }+ LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB2.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB2.file(), null));LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);- prop.put("geo1Status", LibraryProvider.Dictionary.GEODB1.file().exists() ? 1 : 0);- prop.put("geo1ActionLoaded", 1);+ prop.put("geo2Status", LibraryProvider.Dictionary.GEODB2.file().exists() ? 1 : 0);+ prop.put("geo2ActionLoaded", isUpgrade ? 3 : 1);final String nav = sb.getConfig("search.navigation", "");if (nav.indexOf("location") < 0) sb.setConfig("search.navigation", "location,"+nav);} catch (final MalformedURLException e) {ConcurrentLog.logException(e);- prop.put("geo1ActionLoaded", 2);- prop.put("geo1ActionLoaded_error", e.getMessage());+ prop.put("geo2ActionLoaded", 2);+ prop.put("geo2ActionLoaded_error", e.getMessage());} catch (final IOException e) {ConcurrentLog.logException(e);- prop.put("geo1ActionLoaded", 2);- prop.put("geo1ActionLoaded_error", e.getMessage());+ prop.put("geo2ActionLoaded", 2);+ prop.put("geo2ActionLoaded_error", e.getMessage());}}++ if (post.containsKey("geo0Remove")) {+ FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.file());+ FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB0.fileDisabled());+ LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB0.nickname);+ prop.put("geo0ActionRemoved", 1);+ }+ if (post.containsKey("geo0Deactivate")) {+ LibraryProvider.Dictionary.GEODB0.file().renameTo(LibraryProvider.Dictionary.GEODB0.fileDisabled());+ LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB0.nickname);+ prop.put("geo0ActionDeactivated", 1);+ }+if (post.containsKey("geo1Remove")) {FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.file());FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB1.fileDisabled());@@ -246,11 +273,24 @@ public class DictionaryLoader_p {prop.put("geo1ActionDeactivated", 1);}- if (post.containsKey("geo1Activate")) {- LibraryProvider.Dictionary.GEODB1.fileDisabled().renameTo(LibraryProvider.Dictionary.GEODB1.file());- LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB1.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB1.file(), null));+ if (post.containsKey("geo2Remove")) {+ FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB2.file());+ FileUtils.deletedelete(LibraryProvider.Dictionary.GEODB2.fileDisabled());+ LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB2.nickname);+ prop.put("geo2ActionRemoved", 1);+ }++ if (post.containsKey("geo2Deactivate")) {+ LibraryProvider.Dictionary.GEODB2.file().renameTo(LibraryProvider.Dictionary.GEODB2.fileDisabled());+ LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB2.nickname);+ prop.put("geo2ActionDeactivated", 1);+ }++ if (post.containsKey("geo2Activate")) {+ LibraryProvider.Dictionary.GEODB2.fileDisabled().renameTo(LibraryProvider.Dictionary.GEODB2.file());+ LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEODB2.nickname, new OpenGeoDBLocation(LibraryProvider.Dictionary.GEODB2.file(), null));LibraryProvider.autotagging.addPlaces(LibraryProvider.geoLoc);- prop.put("geo1ActionActivated", 1);+ prop.put("geo2ActionActivated", 1);}// DRW0diff --git a/source/net/yacy/document/LibraryProvider.java b/source/net/yacy/document/LibraryProvider.javaindex 0c324d0ff..0b8a202db 100644--- a/source/net/yacy/document/LibraryProvider.java+++ b/source/net/yacy/document/LibraryProvider.java@@ -52,6 +52,8 @@ import net.yacy.crawler.retrieval.URLRewriterLibrary;import net.yacy.kelondro.util.FileUtils;public class LibraryProvider {++ private final static ConcurrentLog LOG = new ConcurrentLog(LibraryProvider.class.getSimpleName());public static final String path_to_source_dictionaries = "source";public static final String path_to_did_you_mean_dictionaries = "didyoumean";@@ -69,8 +71,12 @@ public class LibraryProvider {private static File dictRoot = null;public static enum Dictionary {+ /** Old OpenGeoDB dump */GEODB0( "geo0", "http://downloads.sourceforge.net/project/opengeodb/Data/0.2.5a/opengeodb-0.2.5a-UTF8-sql.gz" ),+ /** Old OpenGeoDB dump */GEODB1( "geo1", "http://fa-technik.adfc.de/code/opengeodb/dump/opengeodb-02624_2011-10-17.sql.gz" ),+ /** Latest (2017) OpenGeoDB dump */+ GEODB2( "geo2", "http://fa-technik.adfc.de/code/opengeodb/dump/opengeodb-02628_2017-02-07.sql.gz" ),GEON0( "geon0", "http://download.geonames.org/export/dump/cities1000.zip" ),GEON1( "geon1", "http://download.geonames.org/export/dump/cities5000.zip" ),GEON2( "geon2", "http://download.geonames.org/export/dump/cities15000.zip" ),@@ -130,18 +136,24 @@ public class LibraryProvider {}public static void integrateOpenGeoDB() {+ final File geo2 = Dictionary.GEODB2.file();final File geo1 = Dictionary.GEODB1.file();final File geo0 = Dictionary.GEODB0.file();- if ( geo1.exists() ) {- if ( geo0.exists() ) {- geo0.renameTo(Dictionary.GEODB0.fileDisabled());+ if (geo2.exists()) {+ if (geo1.exists() && !geo1.renameTo(Dictionary.GEODB1.fileDisabled())) {+ LOG.warn("Could not rename file " + geo1 + " to " + Dictionary.GEODB1.fileDisabled());+ }+ if (geo0.exists() && !geo0.renameTo(Dictionary.GEODB0.fileDisabled())) {+ LOG.warn("Could not rename file " + geo0 + " to " + Dictionary.GEODB0.fileDisabled());+ }+ geoLoc.activateLocation(Dictionary.GEODB2.nickname, new OpenGeoDBLocation(geo2, dymLib));+ } else if ( geo1.exists() ) {+ if ( geo0.exists() && !geo0.renameTo(Dictionary.GEODB0.fileDisabled())) {+ LOG.warn("Could not rename file " + geo0 + " to " + Dictionary.GEODB0.fileDisabled());}geoLoc.activateLocation(Dictionary.GEODB1.nickname, new OpenGeoDBLocation(geo1, dymLib));- return;- }- if ( geo0.exists() ) {+ } else if ( geo0.exists() ) {geoLoc.activateLocation(Dictionary.GEODB0.nickname, new OpenGeoDBLocation(geo0, dymLib));- return;}}