Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
fix for json importer
Michael Peter Christena857e3d3d551
diff --git a/source/net/yacy/cora/date/AbstractFormatter.java b/source/net/yacy/cora/date/AbstractFormatter.javaindex 932fae059..e2c834720 100644--- a/source/net/yacy/cora/date/AbstractFormatter.java+++ b/source/net/yacy/cora/date/AbstractFormatter.java@@ -25,9 +25,14 @@package net.yacy.cora.date;import java.text.ParseException;+import java.text.SimpleDateFormat;import java.util.Calendar;import java.util.Date;+import java.util.HashMap;+import java.util.Locale;+import java.util.Map;import java.util.TimeZone;+import java.util.regex.Pattern;public abstract class AbstractFormatter implements DateFormatter {@@ -37,7 +42,7 @@ public abstract class AbstractFormatter implements DateFormatter {static {UTCCalendar.setTimeZone(UTCtimeZone);}-+// staticspublic final static long secondMillis = 1000;public final static long minuteMillis = 60 * secondMillis;@@ -49,11 +54,58 @@ public abstract class AbstractFormatter implements DateFormatter {protected long last_time;protected String last_format;-+@Overridepublic abstract Calendar parse(String s, int timezoneOffset) throws ParseException;@Overridepublic abstract String format(final Date date);@Overridepublic abstract String format();++ private static final HashMap<Pattern, SimpleDateFormat> DATE_FORMAT_REGEXPS = new HashMap<Pattern, SimpleDateFormat>() {+ private static final long serialVersionUID = 1321140786174228717L;+ {+ put(Pattern.compile("^\\d{8}$"), new SimpleDateFormat("yyyyMMdd", Locale.US));+ put(Pattern.compile("^\\d{1,2}-\\d{1,2}-\\d{4}$"), new SimpleDateFormat("dd-MM-yyyy", Locale.US));+ put(Pattern.compile("^\\d{4}-\\d{1,2}-\\d{1,2}$"), new SimpleDateFormat("yyyy-MM-dd", Locale.US));+ put(Pattern.compile("^\\d{1,2}/\\d{1,2}/\\d{4}$"), new SimpleDateFormat("MM/dd/yyyy", Locale.US));+ put(Pattern.compile("^\\d{4}/\\d{1,2}/\\d{1,2}$"), new SimpleDateFormat("yyyy/MM/dd", Locale.US));+ put(Pattern.compile("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}$"), new SimpleDateFormat("dd MMM yyyy", Locale.US));+ put(Pattern.compile("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}$"), new SimpleDateFormat("dd MMMM yyyy", Locale.US));+ put(Pattern.compile("^\\d{12}$"), new SimpleDateFormat("yyyyMMddHHmm", Locale.US));+ put(Pattern.compile("^\\d{8}\\s\\d{4}$"), new SimpleDateFormat("yyyyMMdd HHmm", Locale.US));+ put(Pattern.compile("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}$"), new SimpleDateFormat("dd-MM-yyyy HH:mm", Locale.US));+ put(Pattern.compile("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}$"), new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.US));+ put(Pattern.compile("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}$"), new SimpleDateFormat("MM/dd/yyyy HH:mm", Locale.US));+ put(Pattern.compile("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}$"), new SimpleDateFormat("yyyy/MM/dd HH:mm", Locale.US));+ put(Pattern.compile("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}$"), new SimpleDateFormat("dd MMM yyyy HH:mm", Locale.US));+ put(Pattern.compile("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}$"), new SimpleDateFormat("dd MMMM yyyy HH:mm", Locale.US));+ put(Pattern.compile("^\\d{14}$"), new SimpleDateFormat("yyyyMMddHHmmss", Locale.US));+ put(Pattern.compile("^\\d{8}\\s\\d{6}$"), new SimpleDateFormat("yyyyMMdd HHmmss", Locale.US));+ put(Pattern.compile("^\\d{1,2}-\\d{1,2}-\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$"), new SimpleDateFormat("dd-MM-yyyy HH:mm:ss", Locale.US));+ put(Pattern.compile("^\\d{4}-\\d{1,2}-\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$"), new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US));+ put(Pattern.compile("^\\d{1,2}/\\d{1,2}/\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$"), new SimpleDateFormat("MM/dd/yyyy HH:mm:ss", Locale.US));+ put(Pattern.compile("^\\d{4}/\\d{1,2}/\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}$"), new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.US));+ put(Pattern.compile("^\\d{1,2}\\s[a-z]{3}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$"), new SimpleDateFormat("dd MMM yyyy HH:mm:ss", Locale.US));+ put(Pattern.compile("^\\d{1,2}\\s[a-z]{4,}\\s\\d{4}\\s\\d{1,2}:\\d{2}:\\d{2}$"), new SimpleDateFormat("dd MMMM yyyy HH:mm:ss", Locale.US));+ put(Pattern.compile("^[a-z]{3}\\s[a-z]{3}\\s\\d{1,2}\\s\\d{1,2}:\\d{2}:\\d{2}\\s[a-z]{4,}\\s\\d{4}$"), new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy", Locale.US));+ }};++ @Override+ public Date parse(String s) {+ return parseAny(s);+ }++ public static Date parseAny(String s) {+ for (Map.Entry<Pattern, SimpleDateFormat> ps: DATE_FORMAT_REGEXPS.entrySet()) {+ if (ps.getKey().matcher(s.toLowerCase()).matches()) {+ try {+ return ps.getValue().parse(s);+ } catch (ParseException e) {+ }+ }+ }+ return null; // Unknown format.+ }+}diff --git a/source/net/yacy/cora/date/DateFormatter.java b/source/net/yacy/cora/date/DateFormatter.javaindex f929534d1..261e0fa97 100644--- a/source/net/yacy/cora/date/DateFormatter.java+++ b/source/net/yacy/cora/date/DateFormatter.java@@ -30,6 +30,7 @@ import java.util.Date;public interface DateFormatter {+ public Date parse(String s);public Calendar parse(String s, int timezoneOffset) throws ParseException;public String format(final Date date);public String format();diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.javaindex 4038ebb63..37616845e 100644--- a/source/net/yacy/search/Switchboard.java+++ b/source/net/yacy/search/Switchboard.java@@ -97,6 +97,7 @@ import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrInputDocument;import org.apache.solr.core.SolrCore;import org.apache.solr.search.SyntaxError;+import org.eclipse.jetty.http.DateParser;import org.json.JSONArray;import org.json.JSONException;import org.json.JSONObject;@@ -108,7 +109,9 @@ import com.google.common.io.Files;import net.yacy.contentcontrol.ContentControlFilterUpdateThread;import net.yacy.contentcontrol.SMWListSyncThread;+import net.yacy.cora.date.AbstractFormatter;import net.yacy.cora.date.GenericFormatter;+import net.yacy.cora.date.ISO8601Formatter;import net.yacy.cora.document.WordCache;import net.yacy.cora.document.analysis.Classification;import net.yacy.cora.document.encoding.ASCII;@@ -121,6 +124,7 @@ import net.yacy.cora.document.id.DigestURL;import net.yacy.cora.document.id.MultiProtocolURL;import net.yacy.cora.federate.solr.FailCategory;import net.yacy.cora.federate.solr.Ranking;+import net.yacy.cora.federate.solr.SolrType;import net.yacy.cora.federate.solr.connector.ShardSelection;import net.yacy.cora.federate.solr.connector.SolrConnector.LoadTimeURL;import net.yacy.cora.federate.solr.instance.EmbeddedInstance;@@ -2199,6 +2203,7 @@ public final class Switchboard extends serverSwitch {// see https://github.com/yacy/yacy_grid_parser/blob/master/README.mdFileInputStream fis = null;BufferedReader br = null;+ VocabularyScraper scraper = new VocabularyScraper();try {fis = new FileInputStream(infile);InputStream is = new BufferedInputStream(fis);@@ -2256,8 +2261,11 @@ public final class Switchboard extends serverSwitch {schema.add(surrogate, list);}} else {- // patch yacy grid altered schema (yacy grid does not have IDs any more, but they can be re-computed here)- if (key.equals("url_s")) {+ CollectionSchema ctype = null;+ try {ctype = CollectionSchema.valueOf(key);} catch (IllegalArgumentException e) {}+ if (key.equals("url_s") || key.equals("sku")) {+ ctype = CollectionSchema.sku;+ // patch yacy grid altered schema (yacy grid does not have IDs any more, but they can be re-computed here)DigestURL durl = new DigestURL(o.toString());String id = ASCII.String(durl.hash());surrogate.setField(CollectionSchema.sku.getSolrFieldName(), durl.toNormalform(true));@@ -2267,11 +2275,29 @@ public final class Switchboard extends serverSwitch {DigestURL durl = new DigestURL(o.toString());String id = ASCII.String(durl.hash());surrogate.setField(CollectionSchema.referrer_id_s.getSolrFieldName(), id);+ } else if (ctype != null && ctype.getType() == SolrType.date) {+ // patch date into something that Solr can understand+ String d = o.toString(); // i.e. Wed Apr 01 02:00:00 CEST 2020+ Date dd = d == null || d.length() == 0 ? null : AbstractFormatter.parseAny(d);+ if (dd != null) surrogate.setField(ctype.getSolrFieldName(), ISO8601Formatter.FORMATTER.format(dd)); // solr dateTime is ISO8601 format} else {surrogate.setField(key, o.toString());}}}++ // enrich the surrogate+ final String id = (String) surrogate.getFieldValue(CollectionSchema.id.getSolrFieldName());+ final String text = (String) surrogate.getFieldValue(CollectionSchema.text_t.getSolrFieldName());+ final DigestURL rootURL = new DigestURL((String) surrogate.getFieldValue(CollectionSchema.sku.getSolrFieldName()), ASCII.getBytes(id));+ if (text != null && text.length() > 0 && id != null ) {+ // run the tokenizer on the text to get vocabularies and synonyms+ final Tokenizer tokenizer = new Tokenizer(rootURL, text, LibraryProvider.dymLib, true, scraper);+ final Map<String, Set<String>> facets = Document.computeGenericFacets(tokenizer.tags());+ // overwrite the given vocabularies and synonyms with new computed ones+ Switchboard.this.index.fulltext().getDefaultConfiguration().enrich(surrogate, tokenizer.synonyms(), facets);+ }+Switchboard.this.index.putDocument(surrogate);}br.close();