Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

refactor xlsParser to include Excel file attribute (like author) in parser result doc. Similar to ppt and doc parser, completing a TODO in xlsParser.

regerebde21079ade
diff --git a/source/net/yacy/document/parser/xlsParser.java b/source/net/yacy/document/parser/xlsParser.java
index 749019134..2e0e9cb52 100644
--- a/source/net/yacy/document/parser/xlsParser.java
+++ b/source/net/yacy/document/parser/xlsParser.java
@@ -27,23 +27,22 @@
package net.yacy.document.parser;
+import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
-import java.util.Date;
+import java.util.ArrayList;
+import java.util.List;
import net.yacy.cora.document.id.DigestURL;
-import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.cora.document.id.MultiProtocolURL;
+import net.yacy.cora.util.CommonPattern;
import net.yacy.document.AbstractParser;
import net.yacy.document.Document;
import net.yacy.document.Parser;
import net.yacy.document.VocabularyScraper;
-import org.apache.poi.hssf.eventusermodel.HSSFEventFactory;
-import org.apache.poi.hssf.eventusermodel.HSSFListener;
-import org.apache.poi.hssf.eventusermodel.HSSFRequest;
-import org.apache.poi.hssf.record.NumberRecord;
-import org.apache.poi.hssf.record.Record;
-import org.apache.poi.hssf.record.SSTRecord;
+import org.apache.poi.hpsf.SummaryInformation;
+import org.apache.poi.hssf.extractor.ExcelExtractor;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
@@ -76,113 +75,52 @@ public class xlsParser extends AbstractParser implements Parser {
final int timezoneOffset,
final InputStream source) throws Parser.Failure,
InterruptedException {
- return new XLSHSSFListener().parse(location, mimeType, charset, source);
- }
-
- public class XLSHSSFListener implements HSSFListener {
-
- //StringBuilder for parsed text
- private final StringBuilder sbFoundStrings;
-
- public XLSHSSFListener() {
- this.sbFoundStrings = new StringBuilder(100);
- }
-
- /*
- * parses the source documents and returns a Document containing
- * all extracted information about the parsed document
- */
- public Document[] parse(final DigestURL location, final String mimeType,
- @SuppressWarnings("unused") final String charset, final InputStream source) throws Parser.Failure,
- InterruptedException {
- try {
-
- //create a new org.apache.poi.poifs.filesystem.Filesystem
- final POIFSFileSystem poifs = new POIFSFileSystem(source);
- //get the Workbook (excel part) stream in a InputStream
- final InputStream din = poifs.createDocumentInputStream("Workbook");
- //construct out HSSFRequest object
- final HSSFRequest req = new HSSFRequest();
- //lazy listen for ALL records with the listener shown above
- req.addListenerForAllRecords(this);
- //create our event factory
- final HSSFEventFactory factory = new HSSFEventFactory();
- //process our events based on the document input stream
- factory.processEvents(req, din);
- //close our document input stream (don't want to leak these!)
- din.close();
-
- //now the parsed strings are in the StringBuilder, now convert them to a String
- final String contents = this.sbFoundStrings.toString().trim();
-
- /*
- * create the plasmaParserDocument for the database
- * and set shortText and bodyText properly
- */
- return new Document[]{new Document(
- location,
- mimeType,
- StandardCharsets.UTF_8.name(),
- this,
- null,
- null,
- singleList(location.getFile()),
- null, // TODO: AUTHOR
- "", // TODO: publisher
- null,
- null,
- 0.0d, 0.0d,
- contents,
- null,
- null,
- null,
- false,
- new Date())};
- } catch (final Exception e) {
- if (e instanceof InterruptedException) throw (InterruptedException) e;
-
- /*
- * an unexpected error occurred, log it and throw a Parser.Failure
- */
- ConcurrentLog.logException(e);
- final String errorMsg = "Unable to parse the xls document '" + location + "':" + e.getMessage();
- throw new Parser.Failure(errorMsg, location);
- }
- }
-
- @Override
- public void processRecord(final Record record) {
- SSTRecord sstrec = null;
- switch (record.getSid()){
- case NumberRecord.sid: {
- final NumberRecord numrec = (NumberRecord) record;
- this.sbFoundStrings.append(numrec.getValue());
- break;
- }
- //unique string records
- case SSTRecord.sid: {
- sstrec = (SSTRecord) record;
- for (int k = 0; k < sstrec.getNumUniqueStrings(); k++){
- this.sbFoundStrings.append( sstrec.getString(k) );
-
- //add line seperator
- this.sbFoundStrings.append( "\n" );
- }
- break;
- }
- /*
- case LabelSSTRecord.sid: {
- final LabelSSTRecord lsrec = (LabelSSTRecord)record;
- sbFoundStrings.append( sstrec.getString(lsrec.getSSTIndex()) );
- break;
- }
- */
- }
-
- //add line seperator
- this.sbFoundStrings.append( "\n" );
+ try {
+ //create a new org.apache.poi.poifs.filesystem.Filesystem
+ final POIFSFileSystem poifs = new POIFSFileSystem(source);
+ ExcelExtractor exceldoc = new ExcelExtractor(poifs);
+ exceldoc.setIncludeSheetNames(false); // exclude sheet names from getText() as also empty sheet names are returned
+
+ SummaryInformation sumInfo = exceldoc.getSummaryInformation();
+ String title = sumInfo.getTitle();
+ if (title == null || title.isEmpty()) title = MultiProtocolURL.unescape(location.getFileName());
+
+ final String subject = sumInfo.getSubject();
+ List<String> descriptions = new ArrayList<String>();
+ if (subject != null && !subject.isEmpty()) descriptions.add(subject);
+
+ // get keywords (for yacy as array)
+ final String keywords = sumInfo.getKeywords();
+ final String[] keywlist;
+ if (keywords != null && !keywords.isEmpty()) {
+ keywlist = CommonPattern.COMMA.split(keywords);
+ } else keywlist = null;
+
+ Document[] retdocs = new Document[]{new Document(
+ location,
+ mimeType,
+ StandardCharsets.UTF_8.name(),
+ this,
+ null,
+ keywlist,
+ singleList(title),
+ sumInfo.getAuthor(),
+ exceldoc.getDocSummaryInformation().getCompany(),
+ null,
+ descriptions,
+ 0.0d, 0.0d,
+ exceldoc.getText(),
+ null,
+ null,
+ null,
+ false,
+ sumInfo.getLastSaveDateTime())};
+
+ return retdocs;
+
+ } catch (IOException ex1) {
+ throw new Parser.Failure(ex1.getMessage(), location);
}
}
-
}
diff --git a/test/java/net/yacy/document/parser/xlsParserTest.java b/test/java/net/yacy/document/parser/xlsParserTest.java
new file mode 100644
index 000000000..eff6b82cb
--- /dev/null
+++ b/test/java/net/yacy/document/parser/xlsParserTest.java
@@ -0,0 +1,57 @@
+package net.yacy.document.parser;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import net.yacy.cora.document.id.AnchorURL;
+import net.yacy.document.AbstractParser;
+import net.yacy.document.Document;
+import net.yacy.document.VocabularyScraper;
+import static org.hamcrest.CoreMatchers.containsString;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class xlsParserTest {
+
+ /**
+ * Test of parse method, of class xlsParser.
+ */
+ @Test
+ public void testParse() throws Exception {
+ final String[][] testFiles = new String[][]{
+ // meaning: filename in test/parsertest, mimetype, title, creator, description,
+ new String[]{"umlaute_linux.xls", "application/msexcel", "In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen", "", ""},
+ new String[]{"umlaute_mac.xls", "application/msexcel", "In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen", "", ""},
+ new String[]{"umlaute_windows.xls", "application/msexcel", "In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen", "afieg", ""}
+
+ };
+
+ for (final String[] testFile : testFiles) {
+
+ final String filename = "test/parsertest/" + testFile[0];
+ final File file = new File(filename);
+ final String mimetype = testFile[1];
+ final AnchorURL url = new AnchorURL("http://localhost/" + filename);
+
+ AbstractParser p = new xlsParser();
+ final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, new FileInputStream(file));
+ for (final Document doc : docs) {
+ final Reader content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
+ final StringBuilder str = new StringBuilder();
+ int c;
+ while ((c = content.read()) != -1) {
+ str.append((char) c);
+ }
+
+ System.out.println("Parsed " + filename + ": " + str);
+ assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen"));
+ assertThat(doc.dc_creator(), containsString(testFile[3]));
+ if (testFile[4].length() > 0) {
+ assertThat(doc.dc_description()[0], containsString(testFile[4]));
+ }
+ }
+ }
+ }
+
+}