The Unified and Autonomous Development Platform
Store parsed symbols in Lucene index for better performance
robin shine28215f746809
diff --git a/commons.git/src/main/java/com/gitplex/commons/git/Blob.java b/commons.git/src/main/java/com/gitplex/commons/git/Blob.javaindex aa348fdfff..90f0628143 100644--- a/commons.git/src/main/java/com/gitplex/commons/git/Blob.java+++ b/commons.git/src/main/java/com/gitplex/commons/git/Blob.java@@ -7,6 +7,7 @@ import java.util.List;import javax.annotation.Nullable;import org.apache.tika.mime.MediaType;+import org.eclipse.jgit.lib.ObjectId;import com.gitplex.commons.util.ContentDetector;import com.google.common.base.Charsets;@@ -16,6 +17,8 @@ public class Blob {private final BlobIdent ident;+ private final ObjectId blobId;+private final byte[] bytes;private final long size;@@ -24,12 +27,13 @@ public class Blob {private transient Optional<Text> optionalText;- public Blob(BlobIdent ident, byte[] bytes) {- this(ident, bytes, bytes.length);+ public Blob(BlobIdent ident, ObjectId id, byte[] bytes) {+ this(ident, id, bytes, bytes.length);}- public Blob(BlobIdent ident, byte[] bytes, long size) {+ public Blob(BlobIdent ident, ObjectId blobId, byte[] bytes, long size) {this.ident = ident;+ this.blobId = blobId;this.bytes = bytes;this.size = size;}@@ -38,6 +42,10 @@ public class Blob {return ident;}+ public ObjectId getBlobId() {+ return blobId;+ }+public byte[] getBytes() {return bytes;}diff --git a/server.core/src/main/java/com/gitplex/server/core/entity/Depot.java b/server.core/src/main/java/com/gitplex/server/core/entity/Depot.javaindex 25b16ea5b1..38ea4a2b49 100644--- a/server.core/src/main/java/com/gitplex/server/core/entity/Depot.java+++ b/server.core/src/main/java/com/gitplex/server/core/entity/Depot.java@@ -494,18 +494,18 @@ public class Depot extends AbstractEntity implements AccountBelonging {defaultBranch = null;}- private Blob readBlob(ObjectLoader objectLoader, BlobIdent ident) {+ private Blob readBlob(ObjectLoader objectLoader, BlobIdent ident, ObjectId id) {long blobSize = objectLoader.getSize();if (blobSize > MAX_READ_BLOB_SIZE) {try (InputStream is = objectLoader.openStream()) {byte[] bytes = new byte[MAX_READ_BLOB_SIZE];is.read(bytes);- return new Blob(ident, bytes, blobSize);+ return new Blob(ident, id, bytes, blobSize);} catch (IOException e) {throw new RuntimeException(e);}} else {- return new Blob(ident, objectLoader.getCachedBytes());+ return new Blob(ident, id, objectLoader.getCachedBytes());}}@@ -546,17 +546,18 @@ public class Depot extends AbstractEntity implements AccountBelonging {RevTree revTree = revWalk.parseCommit(revId).getTree();TreeWalk treeWalk = TreeWalk.forPath(getRepository(), blobIdent.path, revTree);if (treeWalk != null) {+ ObjectId blobId = treeWalk.getObjectId(0);if (blobIdent.isGitLink()) {String url = getSubmodules(blobIdent.revision).get(blobIdent.path);if (url == null)throw new GitObjectNotFoundException("Unable to find submodule '" + blobIdent.path + "' in .gitmodules");- String hash = treeWalk.getObjectId(0).name();- blob = new Blob(blobIdent, new Submodule(url, hash).toString().getBytes());+ String hash = blobId.name();+ blob = new Blob(blobIdent, blobId, new Submodule(url, hash).toString().getBytes());} else if (blobIdent.isTree()) {throw new NotGitFileException("Path '" + blobIdent.path + "' is a tree");} else {- ObjectLoader objectLoader = treeWalk.getObjectReader().open(treeWalk.getObjectId(0));- blob = readBlob(objectLoader, blobIdent);+ ObjectLoader objectLoader = treeWalk.getObjectReader().open(blobId);+ blob = readBlob(objectLoader, blobIdent, blobId);}getBlobCache().put(blobIdent, blob);} else {diff --git a/server.search/src/main/java/com/gitplex/server/search/DefaultIndexManager.java b/server.search/src/main/java/com/gitplex/server/search/DefaultIndexManager.javaindex 635cf29fd5..10a4b096b0 100644--- a/server.search/src/main/java/com/gitplex/server/search/DefaultIndexManager.java+++ b/server.search/src/main/java/com/gitplex/server/search/DefaultIndexManager.java@@ -6,6 +6,7 @@ import static com.gitplex.server.search.FieldConstants.BLOB_NAME;import static com.gitplex.server.search.FieldConstants.BLOB_PATH;import static com.gitplex.server.search.FieldConstants.BLOB_PRIMARY_SYMBOLS;import static com.gitplex.server.search.FieldConstants.BLOB_SECONDARY_SYMBOLS;+import static com.gitplex.server.search.FieldConstants.BLOB_SYMBOL_LIST;import static com.gitplex.server.search.FieldConstants.BLOB_TEXT;import static com.gitplex.server.search.FieldConstants.COMMIT_HASH;import static com.gitplex.server.search.FieldConstants.COMMIT_INDEX_VERSION;@@ -18,13 +19,16 @@ import static com.gitplex.server.search.IndexConstants.NGRAM_SIZE;import java.io.File;import java.io.IOException;+import java.io.Serializable;import java.util.Collection;+import java.util.List;import java.util.concurrent.Callable;import java.util.concurrent.atomic.AtomicReference;import javax.inject.Inject;import javax.inject.Singleton;+import org.apache.commons.lang.SerializationUtils;import org.apache.commons.lang3.StringUtils;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field.Store;@@ -64,6 +68,10 @@ import com.gitplex.commons.loader.Listen;import com.gitplex.commons.loader.ListenerRegistry;import com.gitplex.commons.util.ContentDetector;import com.gitplex.commons.util.concurrent.Prioritized;+import com.gitplex.jsymbol.ExtractException;+import com.gitplex.jsymbol.Symbol;+import com.gitplex.jsymbol.SymbolExtractor;+import com.gitplex.jsymbol.SymbolExtractorRegistry;import com.gitplex.server.core.entity.Depot;import com.gitplex.server.core.event.RefUpdated;import com.gitplex.server.core.manager.BatchWorkManager;@@ -71,10 +79,6 @@ import com.gitplex.server.core.manager.DepotManager;import com.gitplex.server.core.manager.StorageManager;import com.gitplex.server.core.manager.support.BatchWorker;import com.gitplex.server.core.manager.support.IndexResult;-import com.gitplex.jsymbol.ExtractException;-import com.gitplex.jsymbol.Symbol;-import com.gitplex.jsymbol.SymbolExtractor;-import com.gitplex.jsymbol.SymbolExtractorRegistry;import com.google.common.base.Preconditions;import com.google.common.base.Throwables;@@ -263,8 +267,9 @@ public class DefaultIndexManager implements IndexManager {if (extractor != null) {try {- for (Symbol symbol: extractor.extract(content)) {- if (!symbol.isLocal()) {+ List<Symbol> symbols = extractor.extract(content);+ for (Symbol symbol: symbols) {+ if (!symbol.isEffectivelyLocal()) {String fieldValue = symbol.getName();if (fieldValue != null) {fieldValue = fieldValue.toLowerCase();@@ -278,6 +283,8 @@ public class DefaultIndexManager implements IndexManager {}}}+ byte[] bytesOfSymbols = SerializationUtils.serialize((Serializable) symbols);+ document.add(new StoredField(BLOB_SYMBOL_LIST.name(), bytesOfSymbols));} catch (ExtractException e) {logger.debug("Error extracting symbols from blob (hash:" + blobId.name() + ", path:" + blobPath + ")", e);}diff --git a/server.search/src/main/java/com/gitplex/server/search/DefaultSearchManager.java b/server.search/src/main/java/com/gitplex/server/search/DefaultSearchManager.javaindex 6fd8920a8b..a59e9969ac 100644--- a/server.search/src/main/java/com/gitplex/server/search/DefaultSearchManager.java+++ b/server.search/src/main/java/com/gitplex/server/search/DefaultSearchManager.java@@ -1,22 +1,29 @@package com.gitplex.server.search;+import static com.gitplex.server.search.FieldConstants.BLOB_HASH;+import static com.gitplex.server.search.FieldConstants.BLOB_PATH;+import static com.gitplex.server.search.FieldConstants.BLOB_SYMBOL_LIST;+import java.io.IOException;import java.nio.channels.ClosedByInterruptException;import java.util.ArrayList;-import java.util.Collections;import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Set;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.atomic.AtomicReference;import javax.annotation.Nullable;import javax.inject.Inject;import javax.inject.Singleton;+import org.apache.commons.lang.SerializationUtils;import org.apache.lucene.index.AtomicReaderContext;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.search.BooleanClause.Occur;+import org.apache.lucene.search.BooleanQuery;import org.apache.lucene.search.Collector;import org.apache.lucene.search.FieldCache;import org.apache.lucene.search.IndexSearcher;@@ -24,11 +31,18 @@ import org.apache.lucene.search.Scorer;import org.apache.lucene.search.SearcherManager;import org.apache.lucene.store.Directory;import org.apache.lucene.store.FSDirectory;+import org.apache.lucene.util.BytesRef;import org.eclipse.jgit.lib.ObjectId;import org.eclipse.jgit.revwalk.RevTree;import org.eclipse.jgit.revwalk.RevWalk;import org.eclipse.jgit.treewalk.TreeWalk;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;+import com.gitplex.commons.hibernate.Transactional;+import com.gitplex.commons.hibernate.dao.Dao;+import com.gitplex.commons.loader.Listen;+import com.gitplex.jsymbol.Symbol;import com.gitplex.server.core.entity.Depot;import com.gitplex.server.core.event.depot.DepotDeleted;import com.gitplex.server.core.event.lifecycle.SystemStopping;@@ -36,13 +50,12 @@ import com.gitplex.server.core.manager.StorageManager;import com.gitplex.server.search.hit.QueryHit;import com.gitplex.server.search.query.BlobQuery;import com.google.common.base.Throwables;-import com.gitplex.commons.hibernate.Transactional;-import com.gitplex.commons.hibernate.dao.Dao;-import com.gitplex.commons.loader.Listen;@Singletonpublic class DefaultSearchManager implements SearchManager {+ private static final Logger logger = LoggerFactory.getLogger(DefaultSearchManager.class);+private final StorageManager storageManager;private final Dao dao;@@ -81,7 +94,7 @@ public class DefaultSearchManager implements SearchManager {throw Throwables.propagate(e);}}-+@Overridepublic List<QueryHit> search(Depot depot, ObjectId commit, final BlobQuery query)throws InterruptedException {@@ -113,8 +126,8 @@ public class DefaultSearchManager implements SearchManager {if (!checkedBlobPaths.contains(blobPath)) {TreeWalk treeWalk = TreeWalk.forPath(depot.getRepository(), blobPath, revTree);- if (treeWalk != null)- query.collect(treeWalk, hits);+ if (treeWalk != null)+ query.collect(searcher, treeWalk, hits);checkedBlobPaths.add(blobPath);}}@@ -142,11 +155,81 @@ public class DefaultSearchManager implements SearchManager {if (Thread.interrupted())throw new InterruptedException();- Collections.sort(hits);-return hits;}+ @Override+ public List<Symbol> getSymbols(Depot depot, ObjectId blobId, String blobPath) {+ try {+ SearcherManager searcherManager = getSearcherManager(depot);+ if (searcherManager != null) {+ try {+ IndexSearcher searcher = searcherManager.acquire();+ try {+ return getSymbols(searcher, blobId, blobPath);+ } finally {+ searcherManager.release(searcher);+ }+ } catch (IOException e) {+ throw Throwables.propagate(e);+ }+ } else {+ return null;+ }+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }+ }++ @Override+ public List<Symbol> getSymbols(IndexSearcher searcher, ObjectId blobId, String blobPath) {+ BooleanQuery query = new BooleanQuery();+ query.add(BLOB_HASH.query(blobId.name()), Occur.MUST);+ query.add(BLOB_PATH.query(blobPath), Occur.MUST);++ final AtomicReference<List<Symbol>> symbolsRef = new AtomicReference<>(null);+ if (searcher != null) {+ try {+ searcher.search(query, new Collector() {++ private AtomicReaderContext context;++ @Override+ public void setScorer(Scorer scorer) throws IOException {+ }++ @SuppressWarnings("unchecked")+ @Override+ public void collect(int doc) throws IOException {+ BytesRef bytesRef = searcher.doc(context.docBase+doc).getBinaryValue(BLOB_SYMBOL_LIST.name());+ if (bytesRef != null) {+ try {+ symbolsRef.set((List<Symbol>) SerializationUtils.deserialize(bytesRef.bytes));+ } catch (Exception e) {+ logger.error("Error deserializing symbols", e);+ }+ }+ }++ @Override+ public void setNextReader(AtomicReaderContext context) throws IOException {+ this.context = context;+ }++ @Override+ public boolean acceptsDocsOutOfOrder() {+ return true;+ }++ });+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }++ return symbolsRef.get();+ }+@Listenpublic void on(CommitIndexed event) {try {diff --git a/server.search/src/main/java/com/gitplex/server/search/FieldConstants.java b/server.search/src/main/java/com/gitplex/server/search/FieldConstants.javaindex 86c2b0b2bf..348103d84c 100644--- a/server.search/src/main/java/com/gitplex/server/search/FieldConstants.java+++ b/server.search/src/main/java/com/gitplex/server/search/FieldConstants.java@@ -4,8 +4,8 @@ import org.apache.lucene.index.Term;import org.apache.lucene.search.TermQuery;public enum FieldConstants {- BLOB_INDEX_VERSION, BLOB_HASH, BLOB_PATH, BLOB_NAME, BLOB_PRIMARY_SYMBOLS,- BLOB_SECONDARY_SYMBOLS, BLOB_TEXT, COMMIT_INDEX_VERSION, COMMIT_HASH,+ BLOB_INDEX_VERSION, BLOB_HASH, BLOB_PATH, BLOB_NAME, BLOB_SYMBOL_LIST,+ BLOB_PRIMARY_SYMBOLS, BLOB_SECONDARY_SYMBOLS, BLOB_TEXT, COMMIT_INDEX_VERSION, COMMIT_HASH,META, LAST_COMMIT, LAST_COMMIT_HASH, LAST_COMMIT_INDEX_VERSION;public TermQuery query(String value) {diff --git a/server.search/src/main/java/com/gitplex/server/search/SearchManager.java b/server.search/src/main/java/com/gitplex/server/search/SearchManager.javaindex 02231ba5f4..6094f5ac6c 100644--- a/server.search/src/main/java/com/gitplex/server/search/SearchManager.java+++ b/server.search/src/main/java/com/gitplex/server/search/SearchManager.java@@ -2,8 +2,12 @@ package com.gitplex.server.search;import java.util.List;+import javax.annotation.Nullable;++import org.apache.lucene.search.IndexSearcher;import org.eclipse.jgit.lib.ObjectId;+import com.gitplex.jsymbol.Symbol;import com.gitplex.server.core.entity.Depot;import com.gitplex.server.search.hit.QueryHit;import com.gitplex.server.search.query.BlobQuery;@@ -23,4 +27,10 @@ public interface SearchManager {List<QueryHit> search(Depot depot, ObjectId commit, BlobQuery query)throws InterruptedException, TooGeneralQueryException;+ @Nullable+ List<Symbol> getSymbols(Depot depot, ObjectId blobId, String blobPath);++ @Nullable+ List<Symbol> getSymbols(IndexSearcher searcher, ObjectId blobId, String blobPath);+}\ No newline at end of filediff --git a/server.search/src/main/java/com/gitplex/server/search/hit/FileHit.java b/server.search/src/main/java/com/gitplex/server/search/hit/FileHit.javaindex 666ccb4ac2..ade3cbd6b3 100644--- a/server.search/src/main/java/com/gitplex/server/search/hit/FileHit.java+++ b/server.search/src/main/java/com/gitplex/server/search/hit/FileHit.java@@ -42,7 +42,7 @@ public class FileHit extends QueryHit {}@Override- public String getScope() {+ public String getNamespace() {if (getBlobPath().contains("/"))return StringUtils.substringBeforeLast(getBlobPath(), "/");else@@ -53,12 +53,4 @@ public class FileHit extends QueryHit {return matchRange;}- @Override- protected int score() {- if (getBlobPath().contains("/"))- return StringUtils.substringAfterLast(getBlobPath(), "/").length();- else- return getBlobPath().length();- }-}diff --git a/server.search/src/main/java/com/gitplex/server/search/hit/PathHit.java b/server.search/src/main/java/com/gitplex/server/search/hit/PathHit.javaindex 6fe52caf8a..05956613da 100644--- a/server.search/src/main/java/com/gitplex/server/search/hit/PathHit.java+++ b/server.search/src/main/java/com/gitplex/server/search/hit/PathHit.java@@ -37,7 +37,7 @@ public class PathHit extends QueryHit {}@Override- public String getScope() {+ public String getNamespace() {return null;}@@ -45,9 +45,4 @@ public class PathHit extends QueryHit {return matchRange;}- @Override- protected int score() {- return getBlobPath().length();- }-}diff --git a/server.search/src/main/java/com/gitplex/server/search/hit/QueryHit.java b/server.search/src/main/java/com/gitplex/server/search/hit/QueryHit.javaindex 420b26a774..4f0a72ef7f 100644--- a/server.search/src/main/java/com/gitplex/server/search/hit/QueryHit.java+++ b/server.search/src/main/java/com/gitplex/server/search/hit/QueryHit.java@@ -9,7 +9,7 @@ import org.apache.wicket.markup.html.image.Image;import com.gitplex.jsymbol.TokenPosition;-public abstract class QueryHit implements Serializable, Comparable<QueryHit> {+public abstract class QueryHit implements Serializable {private static final long serialVersionUID = 1L;@@ -17,8 +17,6 @@ public abstract class QueryHit implements Serializable, Comparable<QueryHit> {private final TokenPosition tokenPos;- private transient Integer score;-public QueryHit(String blobPath, @Nullable TokenPosition tokenPos) {this.blobPath = blobPath;this.tokenPos = tokenPos;@@ -36,21 +34,8 @@ public abstract class QueryHit implements Serializable, Comparable<QueryHit> {public abstract Component render(String componentId);@Nullable- public abstract String getScope();+ public abstract String getNamespace();public abstract Image renderIcon(String componentId);- protected abstract int score();-- private int getScore() {- if (score == null)- score = score();- return score;- }-- @Override- public int compareTo(QueryHit hit) {- return getScore() - hit.getScore();- }-}diff --git a/server.search/src/main/java/com/gitplex/server/search/hit/SymbolHit.java b/server.search/src/main/java/com/gitplex/server/search/hit/SymbolHit.javaindex 7f09c6fe90..ddfac91ff6 100644--- a/server.search/src/main/java/com/gitplex/server/search/hit/SymbolHit.java+++ b/server.search/src/main/java/com/gitplex/server/search/hit/SymbolHit.java@@ -42,13 +42,8 @@ public class SymbolHit extends QueryHit {}@Override- public String getScope() {+ public String getNamespace() {return symbol.getNamespace();}- @Override- protected int score() {- return symbol.score();- }-}diff --git a/server.search/src/main/java/com/gitplex/server/search/hit/TextHit.java b/server.search/src/main/java/com/gitplex/server/search/hit/TextHit.javaindex 6aeb45fcc4..05216c0dfb 100644--- a/server.search/src/main/java/com/gitplex/server/search/hit/TextHit.java+++ b/server.search/src/main/java/com/gitplex/server/search/hit/TextHit.java@@ -38,16 +38,11 @@ public class TextHit extends QueryHit {}@Override- public String getScope() {+ public String getNamespace() {String fileName = getBlobPath();if (fileName.contains("/"))fileName = StringUtils.substringAfterLast(fileName, "/");return fileName;}- @Override- protected int score() {- return lineContent.length();- }-}diff --git a/server.search/src/main/java/com/gitplex/server/search/query/BlobQuery.java b/server.search/src/main/java/com/gitplex/server/search/query/BlobQuery.javaindex 5100ceccd9..272cffece2 100644--- a/server.search/src/main/java/com/gitplex/server/search/query/BlobQuery.java+++ b/server.search/src/main/java/com/gitplex/server/search/query/BlobQuery.java@@ -8,6 +8,7 @@ import javax.annotation.Nullable;import org.apache.lucene.search.BooleanClause.Occur;import org.apache.lucene.search.BooleanQuery;+import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.Query;import org.apache.lucene.search.WildcardQuery;import org.eclipse.jgit.treewalk.TreeWalk;@@ -34,7 +35,7 @@ public abstract class BlobQuery {return count;}- public abstract void collect(TreeWalk treeWalk, List<QueryHit> hits);+ public abstract void collect(IndexSearcher searcher, TreeWalk treeWalk, List<QueryHit> hits);/*** Get lucene query representation of this querydiff --git a/server.search/src/main/java/com/gitplex/server/search/query/FileQuery.java b/server.search/src/main/java/com/gitplex/server/search/query/FileQuery.javaindex 417ce1e5f4..e7af249f1b 100644--- a/server.search/src/main/java/com/gitplex/server/search/query/FileQuery.java+++ b/server.search/src/main/java/com/gitplex/server/search/query/FileQuery.java@@ -9,6 +9,7 @@ import javax.annotation.Nullable;import org.apache.lucene.index.Term;import org.apache.lucene.search.BooleanClause.Occur;import org.apache.lucene.search.BooleanQuery;+import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.WildcardQuery;import org.eclipse.jgit.treewalk.TreeWalk;@@ -36,7 +37,7 @@ public class FileQuery extends BlobQuery {}@Override- public void collect(TreeWalk treeWalk, List<QueryHit> hits) {+ public void collect(IndexSearcher searcher, TreeWalk treeWalk, List<QueryHit> hits) {String blobPath = treeWalk.getPathString();String blobName = blobPath.substring(blobPath.lastIndexOf('/')+1);if (caseSensitive) {diff --git a/server.search/src/main/java/com/gitplex/server/search/query/PathQuery.java b/server.search/src/main/java/com/gitplex/server/search/query/PathQuery.javaindex 62ef7b798a..4efc8c8db0 100644--- a/server.search/src/main/java/com/gitplex/server/search/query/PathQuery.java+++ b/server.search/src/main/java/com/gitplex/server/search/query/PathQuery.java@@ -9,6 +9,7 @@ import javax.annotation.Nullable;import org.apache.lucene.index.Term;import org.apache.lucene.search.BooleanClause.Occur;import org.apache.lucene.search.BooleanQuery;+import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.WildcardQuery;import org.eclipse.jgit.treewalk.TreeWalk;@@ -27,7 +28,7 @@ public class PathQuery extends BlobQuery {}@Override- public void collect(TreeWalk treeWalk, List<QueryHit> hits) {+ public void collect(IndexSearcher searcher, TreeWalk treeWalk, List<QueryHit> hits) {String blobPath = treeWalk.getPathString();Range range = PathUtils.matchSegments(blobPath, match, true);if (range != null) {diff --git a/server.search/src/main/java/com/gitplex/server/search/query/SourceContext.java b/server.search/src/main/java/com/gitplex/server/search/query/SourceContext.javanew file mode 100644index 0000000000..fcb5806f94--- /dev/null+++ b/server.search/src/main/java/com/gitplex/server/search/query/SourceContext.java@@ -0,0 +1,64 @@+package com.gitplex.server.search.query;++import java.util.HashSet;+import java.util.List;+import java.util.Set;++import com.gitplex.commons.git.Blob;+import com.gitplex.jsymbol.Symbol;+import com.gitplex.jsyntax.Token;+import com.gitplex.jsyntax.Tokenizer;+import com.gitplex.jsyntax.TokenizerRegistry;+import com.google.common.base.CharMatcher;+import com.google.common.base.Preconditions;+import com.google.common.base.Splitter;++public class SourceContext {++ private static final CharMatcher NAMESPACE_SEPARATOR = new CharMatcher() {++ @Override+ public boolean matches(char c) {+ return !Character.isJavaIdentifierPart(c);+ }++ };++ private final Set<String> identifiers;++ public SourceContext(Blob blob) {+ Tokenizer tokenizer = TokenizerRegistry.getTokenizer(blob.getIdent().path);+ if (tokenizer != null) {+ identifiers = new HashSet<>();+ List<String> lines = Preconditions.checkNotNull(blob.getText()).getLines();+ for (List<Token> tokenizedLine: tokenizer.tokenize(lines)) {+ for (Token token: tokenizedLine) {+ if (token.isNotCommentOrString() && !token.isKeyword() && !token.isAtom())+ identifiers.add(token.getText());+ }+ }+ } else {+ identifiers = null;+ }+ }++ public boolean maybeUsing(Symbol symbol) {+ if (identifiers != null) {+ Set<String> parentNames = new HashSet<>();+ Symbol parent = symbol.getParent();+ while (parent != null) {+ if (parent.getName() != null) {+ parentNames.addAll(Splitter+ .on(NAMESPACE_SEPARATOR)+ .omitEmptyStrings()+ .splitToList(parent.getName()));+ }+ parent = parent.getParent();+ }+ return identifiers.containsAll(parentNames);+ } else {+ return true;+ }+ }++}diff --git a/server.search/src/main/java/com/gitplex/server/search/query/SymbolQuery.java b/server.search/src/main/java/com/gitplex/server/search/query/SymbolQuery.javaindex 635f469c03..2187efdc5e 100644--- a/server.search/src/main/java/com/gitplex/server/search/query/SymbolQuery.java+++ b/server.search/src/main/java/com/gitplex/server/search/query/SymbolQuery.java@@ -3,9 +3,7 @@ package com.gitplex.server.search.query;import static com.gitplex.server.search.FieldConstants.BLOB_NAME;import static com.gitplex.server.search.FieldConstants.BLOB_PRIMARY_SYMBOLS;import static com.gitplex.server.search.FieldConstants.BLOB_SECONDARY_SYMBOLS;-import static com.gitplex.server.search.IndexConstants.MAX_INDEXABLE_SIZE;-import java.io.IOException;import java.util.List;import javax.annotation.Nullable;@@ -13,107 +11,94 @@ import javax.annotation.Nullable;import org.apache.lucene.index.Term;import org.apache.lucene.search.BooleanClause.Occur;import org.apache.lucene.search.BooleanQuery;+import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.WildcardQuery;-import org.eclipse.jgit.lib.ObjectLoader;+import org.eclipse.jgit.lib.ObjectId;import org.eclipse.jgit.treewalk.TreeWalk;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;-import com.gitplex.commons.util.ContentDetector;import com.gitplex.commons.util.match.WildcardUtils;-import com.gitplex.server.search.hit.QueryHit;-import com.gitplex.server.search.hit.SymbolHit;-import com.gitplex.jsymbol.ExtractException;import com.gitplex.jsymbol.Range;import com.gitplex.jsymbol.Symbol;-import com.gitplex.jsymbol.SymbolExtractor;-import com.gitplex.jsymbol.SymbolExtractorRegistry;+import com.gitplex.server.core.GitPlex;+import com.gitplex.server.search.SearchManager;+import com.gitplex.server.search.hit.QueryHit;+import com.gitplex.server.search.hit.SymbolHit;import com.google.common.base.Splitter;public class SymbolQuery extends BlobQuery {- private static final Logger logger = LoggerFactory.getLogger(SymbolQuery.class);-private final String term;private final String excludeTerm;+ private final String excludeBlobPath;+private final boolean primary;private final boolean caseSensitive;private final String fileNames;- public SymbolQuery(String term, @Nullable String excludeTerm, boolean primary,- boolean caseSensitive, @Nullable String directory, @Nullable String fileNames,- int count) {+ private final SourceContext sourceContext;++ public SymbolQuery(String term, @Nullable String excludeTerm, @Nullable String excludeBlobPath,+ boolean primary, boolean caseSensitive, @Nullable String directory, @Nullable String fileNames,+ @Nullable SourceContext sourceContext, int count) {super(directory, count);this.term = term;this.excludeTerm = excludeTerm;+ this.excludeBlobPath = excludeBlobPath;this.primary = primary;this.caseSensitive = caseSensitive;this.fileNames = fileNames;+ this.sourceContext = sourceContext;}-+@Override- public void collect(TreeWalk treeWalk, List<QueryHit> hits) {+ public void collect(IndexSearcher searcher, TreeWalk treeWalk, List<QueryHit> hits) {String blobPath = treeWalk.getPathString();+ ObjectId blobId = treeWalk.getObjectId(0);- SymbolExtractor<Symbol> extractor = SymbolExtractorRegistry.getExtractor(blobPath);- if (extractor != null) {- ObjectLoader objectLoader;- try {- objectLoader = treeWalk.getObjectReader().open(treeWalk.getObjectId(0));- if (objectLoader.getSize() <= MAX_INDEXABLE_SIZE) {- byte[] bytes = objectLoader.getCachedBytes();- String content = ContentDetector.convertToText(bytes, blobPath);- if (content != null) {- try {- for (Symbol symbol: extractor.extract(content)) {- if (hits.size() < getCount()) {- if (primary == symbol.isPrimary() && symbol.getName() != null) {- String normalizedTerm;- if (!caseSensitive)- normalizedTerm = term.toLowerCase();- else- normalizedTerm = term;-- String normalizedSymbolName;- if (!caseSensitive)- normalizedSymbolName = symbol.getName().toLowerCase();- else- normalizedSymbolName = symbol.getName();-- String normalizedExcludeTerm;- if (excludeTerm != null) {- if (!caseSensitive)- normalizedExcludeTerm = excludeTerm.toLowerCase();- else- normalizedExcludeTerm = excludeTerm;- } else {- normalizedExcludeTerm = null;- }- if (WildcardUtils.matchString(normalizedTerm, normalizedSymbolName)- && (normalizedExcludeTerm == null || !normalizedSymbolName.equals(normalizedExcludeTerm))) {- Range matchRange = WildcardUtils.rangeOfMatch(normalizedTerm, normalizedSymbolName);- hits.add(new SymbolHit(blobPath, symbol, matchRange));- }- }- } else {- break;- }- }- } catch (ExtractException e) {- logger.debug("Error extracting symbols from blob (hash:"- + treeWalk.getObjectId(0).getName() + ", path:" + blobPath + ")", e);+ List<Symbol> symbols = GitPlex.getInstance(SearchManager.class).getSymbols(searcher, blobId, blobPath);+ if (symbols != null) {+ for (Symbol symbol: symbols) {+ if (hits.size() < getCount()) {+ if (primary == symbol.isPrimary() && symbol.getName() != null && !symbol.isEffectivelyLocal()) {+ String normalizedTerm;+ if (!caseSensitive)+ normalizedTerm = term.toLowerCase();+ else+ normalizedTerm = term;++ String normalizedSymbolName;+ if (!caseSensitive)+ normalizedSymbolName = symbol.getName().toLowerCase();+ else+ normalizedSymbolName = symbol.getName();++ String normalizedExcludeTerm;+ if (excludeTerm != null) {+ if (!caseSensitive)+ normalizedExcludeTerm = excludeTerm.toLowerCase();+ else+ normalizedExcludeTerm = excludeTerm;+ } else {+ normalizedExcludeTerm = null;+ }+ if (WildcardUtils.matchString(normalizedTerm, normalizedSymbolName)+ && (normalizedExcludeTerm == null || !normalizedSymbolName.equals(normalizedExcludeTerm))+ && (excludeBlobPath == null || !excludeBlobPath.equals(blobPath))+ && (sourceContext == null || sourceContext.maybeUsing(symbol))) {+ Range matchRange = WildcardUtils.rangeOfMatch(normalizedTerm, normalizedSymbolName);+ hits.add(new SymbolHit(blobPath, symbol, matchRange));}}+ } else {+ break;}- } catch (IOException e) {- throw new RuntimeException(e);}- }+ }}@Overridediff --git a/server.search/src/main/java/com/gitplex/server/search/query/TextQuery.java b/server.search/src/main/java/com/gitplex/server/search/query/TextQuery.javaindex 40350f83f5..c01e12de99 100644--- a/server.search/src/main/java/com/gitplex/server/search/query/TextQuery.java+++ b/server.search/src/main/java/com/gitplex/server/search/query/TextQuery.java@@ -15,6 +15,7 @@ import org.apache.commons.lang3.CharUtils;import org.apache.lucene.index.Term;import org.apache.lucene.search.BooleanClause.Occur;import org.apache.lucene.search.BooleanQuery;+import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.WildcardQuery;import org.eclipse.jgit.lib.ObjectLoader;import org.eclipse.jgit.treewalk.TreeWalk;@@ -78,7 +79,7 @@ public class TextQuery extends BlobQuery {}@Override- public void collect(TreeWalk treeWalk, List<QueryHit> hits) {+ public void collect(IndexSearcher searcher, TreeWalk treeWalk, List<QueryHit> hits) {ObjectLoader objectLoader;try {objectLoader = treeWalk.getObjectReader().open(treeWalk.getObjectId(0));diff --git a/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/advanced/AdvancedSearchPanel.java b/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/advanced/AdvancedSearchPanel.javaindex 996c04e181..628dc3bcee 100644--- a/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/advanced/AdvancedSearchPanel.java+++ b/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/advanced/AdvancedSearchPanel.java@@ -231,7 +231,8 @@ public abstract class AdvancedSearchPanel extends Panel {});} else {- SymbolQuery query = new SymbolQuery(validatable.getValue(), null, true, false, null, null, 0);+ SymbolQuery query = new SymbolQuery(validatable.getValue(), null, null, true, false, null,+ null, null, 0);try {query.asLuceneQuery();} catch (TooGeneralQueryException e) {@@ -516,16 +517,16 @@ public abstract class AdvancedSearchPanel extends Panel {public List<QueryHit> query(AdvancedSearchPanel context) throws InterruptedException {SearchManager searchManager = GitPlex.getInstance(SearchManager.class);List<QueryHit> hits;- BlobQuery query = new SymbolQuery(term, null, true, caseSensitive,+ BlobQuery query = new SymbolQuery(term, null, null, true, caseSensitive,context.getDirectory(insideCurrentDir), fileNames,- SearchResultPanel.MAX_QUERY_ENTRIES);+ null, SearchResultPanel.MAX_QUERY_ENTRIES);ObjectId commit = context.depotModel.getObject().getRevCommit(context.revisionModel.getObject());hits = searchManager.search(context.depotModel.getObject(), commit, query);if (hits.size() < SearchResultPanel.MAX_QUERY_ENTRIES) {- query = new SymbolQuery(term, null, false, caseSensitive,+ query = new SymbolQuery(term, null, null, false, caseSensitive,context.getDirectory(insideCurrentDir), fileNames,- SearchResultPanel.MAX_QUERY_ENTRIES - hits.size());+ null, SearchResultPanel.MAX_QUERY_ENTRIES - hits.size());hits.addAll(searchManager.search(context.depotModel.getObject(), commit, query));}diff --git a/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/instant/InstantSearchPanel.java b/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/instant/InstantSearchPanel.javaindex ed1da3cf64..30de0d4149 100644--- a/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/instant/InstantSearchPanel.java+++ b/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/instant/InstantSearchPanel.java@@ -84,14 +84,14 @@ public abstract class InstantSearchPanel extends Panel {try {// first try an exact search against primary symbol to make sure the result// always contains exact match if exists- BlobQuery query = new SymbolQuery(searchInput, null, true, false,- null, null, count);+ BlobQuery query = new SymbolQuery(searchInput, null, null, true, false,+ null, null, null, count);symbolHits.addAll(searchManager.search(depotModel.getObject(), commit, query));// now do wildcard search but exclude the exact match returned aboveif (symbolHits.size() < count) {- query = new SymbolQuery(searchInput+"*", searchInput, true, false,- null, null, count-symbolHits.size());+ query = new SymbolQuery(searchInput+"*", searchInput, null, true, false,+ null, null, null, count-symbolHits.size());symbolHits.addAll(searchManager.search(depotModel.getObject(), commit, query));}@@ -110,14 +110,14 @@ public abstract class InstantSearchPanel extends Panel {// do the same for secondary symbolsif (symbolHits.size() < count) {- query = new SymbolQuery(searchInput, null, false, false,- null, null, count-symbolHits.size());+ query = new SymbolQuery(searchInput, null, null, false, false,+ null, null, null, count-symbolHits.size());symbolHits.addAll(searchManager.search(depotModel.getObject(), commit, query));}if (symbolHits.size() < count) {- query = new SymbolQuery(searchInput+"*", searchInput, false, false,- null, null, count-symbolHits.size());+ query = new SymbolQuery(searchInput+"*", searchInput, null, false, false,+ null, null, null, count-symbolHits.size());symbolHits.addAll(searchManager.search(depotModel.getObject(), commit, query));}@@ -243,7 +243,7 @@ public abstract class InstantSearchPanel extends Panel {};link.add(hit.renderIcon("icon"));link.add(hit.render("label"));- link.add(new Label("scope", hit.getScope()).setVisible(hit.getScope()!=null));+ link.add(new Label("scope", hit.getNamespace()).setVisible(hit.getNamespace()!=null));item.add(link);DepotFilePage.State state = new DepotFilePage.State();@@ -365,12 +365,7 @@ public abstract class InstantSearchPanel extends Panel {}@Override- public String getScope() {- throw new UnsupportedOperationException();- }-- @Override- protected int score() {+ public String getNamespace() {throw new UnsupportedOperationException();}diff --git a/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/result/SearchResultPanel.java b/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/result/SearchResultPanel.javaindex 4620e767d8..d0532dbb72 100644--- a/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/result/SearchResultPanel.java+++ b/server.web/src/main/java/com/gitplex/server/web/component/depotfile/blobsearch/result/SearchResultPanel.java@@ -450,8 +450,8 @@ public abstract class SearchResultPanel extends Panel {add(hit.renderIcon("icon"));add(new Label("lineNo", String.valueOf(hit.getTokenPos().getFromLine()+1) + ":"));add(hit.render("label"));- add(new Label("scope", hit.getScope())- .setVisible(!(hit instanceof TextHit) && hit.getScope()!=null));+ add(new Label("scope", hit.getNamespace())+ .setVisible(!(hit instanceof TextHit) && hit.getNamespace()!=null));if (activeBlobIndex == blobItem.getIndex() && activeHitIndex == hitItem.getIndex())add(AttributeAppender.append("class", " active"));diff --git a/server.web/src/main/java/com/gitplex/server/web/component/depotfile/fileedit/FileEditPanel.java b/server.web/src/main/java/com/gitplex/server/web/component/depotfile/fileedit/FileEditPanel.javaindex 349779b268..6cd155ee53 100644--- a/server.web/src/main/java/com/gitplex/server/web/component/depotfile/fileedit/FileEditPanel.java+++ b/server.web/src/main/java/com/gitplex/server/web/component/depotfile/fileedit/FileEditPanel.java@@ -126,7 +126,7 @@ public abstract class FileEditPanel extends Panel {@Overridepublic Blob getBlob(BlobIdent blobIdent) {if (blobIdent.revision.equals(ObjectId.zeroId().name()))- return new Blob(blobIdent, content.getBytes(Charsets.UTF_8));+ return new Blob(blobIdent, ObjectId.zeroId(), content.getBytes(Charsets.UTF_8));elsereturn depotModel.getObject().getBlob(blobIdent);}diff --git a/server.web/src/main/java/com/gitplex/server/web/component/symboltooltip/SymbolTooltipPanel.java b/server.web/src/main/java/com/gitplex/server/web/component/symboltooltip/SymbolTooltipPanel.javaindex 0f234dfb47..298e6d5c13 100644--- a/server.web/src/main/java/com/gitplex/server/web/component/symboltooltip/SymbolTooltipPanel.java+++ b/server.web/src/main/java/com/gitplex/server/web/component/symboltooltip/SymbolTooltipPanel.java@@ -26,19 +26,25 @@ import org.apache.wicket.request.cycle.RequestCycle;import org.apache.wicket.request.mapper.parameter.PageParameters;import org.apache.wicket.request.resource.PackageResourceReference;import org.apache.wicket.request.resource.ResourceReference;+import org.eclipse.jgit.lib.FileMode;import org.eclipse.jgit.lib.ObjectId;+import com.gitplex.commons.git.Blob;+import com.gitplex.commons.git.BlobIdent;import com.gitplex.commons.wicket.behavior.AbstractPostAjaxBehavior;import com.gitplex.commons.wicket.behavior.RunTaskBehavior;import com.gitplex.commons.wicket.component.PreventDefaultAjaxLink;+import com.gitplex.jsymbol.Symbol;import com.gitplex.server.core.GitPlex;import com.gitplex.server.core.entity.Depot;import com.gitplex.server.core.entity.support.TextRange;import com.gitplex.server.search.IndexConstants;import com.gitplex.server.search.SearchManager;import com.gitplex.server.search.hit.QueryHit;+import com.gitplex.server.search.hit.SymbolHit;import com.gitplex.server.search.query.BlobQuery;import com.gitplex.server.search.query.PathQuery;+import com.gitplex.server.search.query.SourceContext;import com.gitplex.server.search.query.SymbolQuery;import com.gitplex.server.search.query.TextQuery;import com.gitplex.server.web.component.depotfile.blobsearch.result.SearchResultPanel;@@ -53,7 +59,7 @@ public abstract class SymbolTooltipPanel extends Panel {private String revision = "";- private String symbol = "";+ private String symbolName = "";private List<QueryHit> symbolHits = new ArrayList<>();@@ -99,7 +105,7 @@ public abstract class SymbolTooltipPanel extends Panel {CharSequence url = RequestCycle.get().urlFor(DepotFilePage.class, getQueryHitParams(hit));link.add(AttributeAppender.replace("href", url.toString()));link.add(hit.render("label"));- link.add(new Label("scope", hit.getScope()).setVisible(hit.getScope()!=null));+ link.add(new Label("scope", hit.getNamespace()).setVisible(hit.getNamespace()!=null));item.add(link);}@@ -140,8 +146,8 @@ public abstract class SymbolTooltipPanel extends Panel {target.prependJavaScript(script);List<QueryHit> hits;// do this check to avoid TooGeneralQueryException- if (symbol.length() >= IndexConstants.NGRAM_SIZE) {- BlobQuery query = new TextQuery(symbol, false, true, true,+ if (symbolName.length() >= IndexConstants.NGRAM_SIZE) {+ BlobQuery query = new TextQuery(symbolName, false, true, true,null, null, SearchResultPanel.MAX_QUERY_ENTRIES);try {SearchManager searchManager = GitPlex.getInstance(SearchManager.class);@@ -181,31 +187,65 @@ public abstract class SymbolTooltipPanel extends Panel {protected void respond(AjaxRequestTarget target) {IRequestParameters params = RequestCycle.get().getRequest().getPostParameters();revision = params.getParameterValue("revision").toString();- symbol = params.getParameterValue("symbol").toString();- boolean isString = symbol.indexOf('\'') != -1 || symbol.indexOf('"') != -1;+ symbolName = params.getParameterValue("symbol").toString();+ boolean isString = symbolName.indexOf('\'') != -1 || symbolName.indexOf('"') != -1;String charsToStrip = "@'\"./\\";- symbol = StringUtils.stripEnd(StringUtils.stripStart(symbol, charsToStrip), charsToStrip);- symbol = StringUtils.replace(symbol, "\\", "/");+ symbolName = StringUtils.stripEnd(StringUtils.stripStart(symbolName, charsToStrip), charsToStrip);+ symbolName = StringUtils.replace(symbolName, "\\", "/");+ symbolHits.clear();+// do this check to avoid TooGeneralQueryException- if (symbol.length() == 0 || symbol.indexOf('?') != -1 || symbol.indexOf('*') != -1) {- symbolHits = new ArrayList<>();- } else {- try {- BlobQuery query = new SymbolQuery(symbol, null, true, true, null, null, QUERY_ENTRIES);- SearchManager searchManager = GitPlex.getInstance(SearchManager.class);- ObjectId commit = depotModel.getObject().getRevCommit(revision);- symbolHits = searchManager.search(depotModel.getObject(), commit, query);- if (symbolHits.size() < QUERY_ENTRIES) {- query = new SymbolQuery(symbol, null, false, true, null, null, QUERY_ENTRIES - symbolHits.size());- symbolHits.addAll(searchManager.search(depotModel.getObject(), commit, query));- }- if (isString && symbolHits.size() < QUERY_ENTRIES) {- query = new PathQuery(null, symbol, QUERY_ENTRIES - symbolHits.size());- symbolHits.addAll(searchManager.search(depotModel.getObject(), commit, query));+ if (symbolName.length() != 0 && symbolName.indexOf('?') == -1 && symbolName.indexOf('*') == -1) {+ BlobIdent blobIdent = new BlobIdent(revision, getBlobPath(), FileMode.TYPE_FILE);+ Blob blob = depotModel.getObject().getBlob(blobIdent);++ if (symbolHits.size() < QUERY_ENTRIES) {+ List<Symbol> symbols = GitPlex.getInstance(SearchManager.class).getSymbols(depotModel.getObject(),+ blob.getBlobId(), getBlobPath());+ if (symbols != null) {+ for (Symbol symbol: symbols) {+ if (symbolHits.size() < QUERY_ENTRIES+ && symbol.isSearchable()+ && symbolName.equals(symbol.getName())+ && symbol.isPrimary()) {+ symbolHits.add(new SymbolHit(getBlobPath(), symbol, null));+ }+ }+ for (Symbol symbol: symbols) {+ if (symbolHits.size() < QUERY_ENTRIES+ && symbol.isSearchable()+ && symbolName.equals(symbol.getName())+ && !symbol.isPrimary()) {+ symbolHits.add(new SymbolHit(getBlobPath(), symbol, null));+ }+ }}- } catch (InterruptedException e) {- throw new RuntimeException(e);- }+ }++ if (symbolHits.size() < QUERY_ENTRIES) {+ SourceContext sourceContext = new SourceContext(blob);+ try {+ SearchManager searchManager = GitPlex.getInstance(SearchManager.class);+ ObjectId commit = depotModel.getObject().getRevCommit(revision);+ BlobQuery query;+ if (symbolHits.size() < QUERY_ENTRIES) {+ query = new SymbolQuery(symbolName, null, blobIdent.path, true, true, null, null,+ sourceContext, QUERY_ENTRIES);+ symbolHits.addAll(searchManager.search(depotModel.getObject(), commit, query));+ }+ if (symbolHits.size() < QUERY_ENTRIES) {+ query = new SymbolQuery(symbolName, null, blobIdent.path, false, true, null, null,+ sourceContext, QUERY_ENTRIES - symbolHits.size());+ symbolHits.addAll(searchManager.search(depotModel.getObject(), commit, query));+ }+ if (isString && symbolHits.size() < QUERY_ENTRIES) {+ query = new PathQuery(null, symbolName, QUERY_ENTRIES - symbolHits.size());+ symbolHits.addAll(searchManager.search(depotModel.getObject(), commit, query));+ }+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }+ }}target.add(content);String script = String.format("gitplex.server.symboltooltip.doneQuery('%s');", content.getMarkupId());@@ -245,12 +285,12 @@ public abstract class SymbolTooltipPanel extends Panel {DepotFilePage.State state = new DepotFilePage.State();state.blobIdent.revision = revision;state.blobIdent.path = getBlobPath();- state.query = symbol;+ state.query = symbolName;return DepotFilePage.paramsOf(depotModel.getObject(), state);}public String getSymbol() {- return symbol;+ return symbolName;}public String getRevision() {