The Unified and Autonomous Development Platform
For clear purpose: rename ExtractToken as Token and rename ExtractStream as TokenStream.
robin shine23524026c686
diff --git a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractToken.java b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/Token.javasimilarity index 75%rename from commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractToken.javarename to commons.lang/src/main/java/com/gitplex/commons/lang/extractors/Token.javaindex 8d9bd4f532..8f6ebc5eee 100644--- a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractToken.java+++ b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/Token.java@@ -2,15 +2,13 @@ package com.gitplex.commons.lang.extractors;import java.io.Serializable;-import org.antlr.v4.runtime.Token;-import com.gitplex.commons.util.Range;-public class ExtractToken implements Serializable {+public class Token implements Serializable {private static final long serialVersionUID = 1L;- public static final ExtractToken EOF = new ExtractToken(Token.EOF, "", null);+ public static final Token EOF = new Token(org.antlr.v4.runtime.Token.EOF, "", null);private final int type;@@ -18,7 +16,7 @@ public class ExtractToken implements Serializable {private final TokenPosition pos;- public ExtractToken(Token antlrToken) {+ public Token(org.antlr.v4.runtime.Token antlrToken) {type = antlrToken.getType();text = antlrToken.getText();Range range = new Range(antlrToken.getCharPositionInLine(),@@ -26,7 +24,7 @@ public class ExtractToken implements Serializable {pos = new TokenPosition(antlrToken.getLine()-1, range);}- public ExtractToken(int type, String text, TokenPosition pos) {+ public Token(int type, String text, TokenPosition pos) {this.type = type;this.text = text;this.pos = pos;@@ -49,7 +47,7 @@ public class ExtractToken implements Serializable {}public boolean isEof() {- return type == Token.EOF;+ return type == org.antlr.v4.runtime.Token.EOF;}public int getType() {@@ -64,13 +62,13 @@ public class ExtractToken implements Serializable {return pos;}- public ExtractToken checkType(int... expectedTypes) {+ public Token checkType(int... expectedTypes) {if (!is(expectedTypes))throw new UnexpectedTokenException(this);return this;}- public ExtractToken checkText(String expectedText) {+ public Token checkText(String expectedText) {if (!text.equals(expectedText))throw new UnexpectedTokenException(this);return this;diff --git a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractStream.java b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/TokenStream.javasimilarity index 69%rename from commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractStream.javarename to commons.lang/src/main/java/com/gitplex/commons/lang/extractors/TokenStream.javaindex 8726daa4eb..6a0da55091 100644--- a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractStream.java+++ b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/TokenStream.java@@ -7,24 +7,23 @@ import org.antlr.v4.runtime.BaseErrorListener;import org.antlr.v4.runtime.Lexer;import org.antlr.v4.runtime.RecognitionException;import org.antlr.v4.runtime.Recognizer;-import org.antlr.v4.runtime.Token;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-public class ExtractStream {+public class TokenStream {- private static final Logger logger = LoggerFactory.getLogger(ExtractStream.class);+ private static final Logger logger = LoggerFactory.getLogger(TokenStream.class);- private final List<ExtractToken> tokens;+ private final List<Token> tokens;private int pos = -1; // 0-indexed position of current token- public ExtractStream(List<ExtractToken> tokens) {+ public TokenStream(List<Token> tokens) {this.tokens = tokens;- this.tokens.add(ExtractToken.EOF);+ this.tokens.add(Token.EOF);}- public ExtractStream(Lexer lexer, TokenFilter filter) {+ public TokenStream(Lexer lexer, TokenFilter filter) {lexer.removeErrorListeners();lexer.addErrorListener(new BaseErrorListener() {@@ -36,36 +35,36 @@ public class ExtractStream {});tokens = new ArrayList<>();- Token token = lexer.nextToken();- while (token.getType() != Token.EOF) {+ org.antlr.v4.runtime.Token token = lexer.nextToken();+ while (token.getType() != org.antlr.v4.runtime.Token.EOF) {if (filter.accept(token))- tokens.add(new ExtractToken(token));+ tokens.add(new Token(token));token = lexer.nextToken();}- tokens.add(ExtractToken.EOF);+ tokens.add(Token.EOF);}- public ExtractToken at(int pos) {+ public Token at(int pos) {if (pos >=0 && pos < tokens.size())return tokens.get(pos);elsethrow new ExtractException("Invalid token position: " + pos);}- public ExtractToken next() {+ public Token next() {return at(++pos);}- public ExtractToken lookAhead(int ahead) {+ public Token lookAhead(int ahead) {return at(pos+ahead);}- public ExtractToken lookBehind(int behind) {+ public Token lookBehind(int behind) {return at(pos-behind);}- public ExtractToken nextType(int type) {- ExtractToken token = next();+ public Token nextType(int type) {+ Token token = next();while(true) {if (token.is(type))return token;@@ -73,8 +72,8 @@ public class ExtractStream {}}- public ExtractToken nextType(int...types) {- ExtractToken token = next();+ public Token nextType(int...types) {+ Token token = next();while(true) {if (token.is(types))return token;@@ -95,9 +94,9 @@ public class ExtractStream {* @param closeType* @return*/- public ExtractToken nextClosed(int openType, int closeType) {+ public Token nextClosed(int openType, int closeType) {int nestingLevel = 1;- ExtractToken balanced = nextType(openType, closeType);+ Token balanced = nextType(openType, closeType);while (true) {if (balanced.is(closeType)) {if (--nestingLevel == 0)@@ -127,14 +126,14 @@ public class ExtractStream {* @return* list of tokens between startPos and endPos*/- public List<ExtractToken> between(int startPos, int endPos) {- List<ExtractToken> tokens = new ArrayList<>();+ public List<Token> between(int startPos, int endPos) {+ List<Token> tokens = new ArrayList<>();for (int i=startPos; i<=endPos; i++)tokens.add(at(i));return tokens;}- public ExtractToken current() {+ public Token current() {return at(pos);}diff --git a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/UnexpectedTokenException.java b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/UnexpectedTokenException.javaindex f2193f5aca..b75e20fb4c 100644--- a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/UnexpectedTokenException.java+++ b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/UnexpectedTokenException.java@@ -4,7 +4,7 @@ package com.gitplex.commons.lang.extractors;@SuppressWarnings("serial")public class UnexpectedTokenException extends ExtractException {- public UnexpectedTokenException(ExtractToken token) {+ public UnexpectedTokenException(Token token) {super("Unexpected token: " + token);}diff --git a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/java/JavaExtractor.java b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/java/JavaExtractor.javaindex d86c9310ec..2b39d195c4 100644--- a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/java/JavaExtractor.java+++ b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/java/JavaExtractor.java@@ -11,8 +11,8 @@ import javax.annotation.Nullable;import org.antlr.v4.runtime.ANTLRInputStream;import com.gitplex.commons.lang.extractors.AbstractExtractor;-import com.gitplex.commons.lang.extractors.ExtractStream;-import com.gitplex.commons.lang.extractors.ExtractToken;+import com.gitplex.commons.lang.extractors.TokenStream;+import com.gitplex.commons.lang.extractors.Token;import com.gitplex.commons.lang.extractors.Symbol;import com.gitplex.commons.lang.extractors.TokenFilter;import com.gitplex.commons.lang.extractors.TokenPosition;@@ -37,12 +37,12 @@ public class JavaExtractor extends AbstractExtractor {**/public List<Symbol> extract(String text) {- ExtractStream stream = new ExtractStream(+ TokenStream stream = new TokenStream(new JavaLexer(new ANTLRInputStream(text)), TokenFilter.DEFAULT_CHANNEL);List<Symbol> symbols = new ArrayList<>();- ExtractToken token = stream.next();+ Token token = stream.next();while (token.is(AT) && !stream.lookAhead(1).is(INTERFACE)) {skipAnnotation(stream);@@ -88,8 +88,8 @@ public class JavaExtractor extends AbstractExtractor {* @before-token: '{' or ';' (';' occurs inside of a enum definition)* @after-token: '}'*/- private void defineTypeBody(ExtractStream stream, List<Symbol> symbols, TypeDef parent, String packageName) {- ExtractToken token = stream.next();+ private void defineTypeBody(TokenStream stream, List<Symbol> symbols, TypeDef parent, String packageName) {+ Token token = stream.next();while(true) {while (token.is(SEMI))token = stream.next();@@ -153,9 +153,9 @@ public class JavaExtractor extends AbstractExtractor {* fields to be consumed**/- private List<Pair<ExtractToken, String>> skipValue(ExtractStream stream) {- List<Pair<ExtractToken, String>> fieldInfos = new ArrayList<>();- ExtractToken token = stream.current();+ private List<Pair<Token, String>> skipValue(TokenStream stream) {+ List<Pair<Token, String>> fieldInfos = new ArrayList<>();+ Token token = stream.current();if (token.is(LBRACE)) { // array valuestream.nextClosed(LBRACE, RBRACE);stream.next();@@ -171,7 +171,7 @@ public class JavaExtractor extends AbstractExtractor {break;} else { // token is ','token.checkType(COMMA);- List<Pair<ExtractToken, String>> subsequentFieldTokens = assumeFieldSeparator(stream);+ List<Pair<Token, String>> subsequentFieldTokens = assumeFieldSeparator(stream);if (subsequentFieldTokens != null) { // assumption is correctfieldInfos.addAll(subsequentFieldTokens);break;@@ -196,13 +196,13 @@ public class JavaExtractor extends AbstractExtractor {* null if assumption is incorrect**/- private List<Pair<ExtractToken, String>> assumeFieldSeparator(ExtractStream stream) {+ private List<Pair<Token, String>> assumeFieldSeparator(TokenStream stream) {while (true) {- ExtractToken token = stream.next();+ Token token = stream.next();if (token.is(Identifier)) {- List<Pair<ExtractToken, String>> fieldInfos = new ArrayList<>();+ List<Pair<Token, String>> fieldInfos = new ArrayList<>();stream.next();- fieldInfos.add(new Pair<ExtractToken, String>(token, skipDims(stream)));+ fieldInfos.add(new Pair<Token, String>(token, skipDims(stream)));token = stream.current();if (token.is(ASSIGN)) { // assumption convincedstream.next();@@ -211,7 +211,7 @@ public class JavaExtractor extends AbstractExtractor {} else if (token.is(SEMI)) { // assumption convincedreturn fieldInfos;} else if (token.is(COMMA)) { // assumption still not convinced- List<Pair<ExtractToken, String>> subsequentFieldInfos = assumeFieldSeparator(stream);+ List<Pair<Token, String>> subsequentFieldInfos = assumeFieldSeparator(stream);if (subsequentFieldInfos != null) {fieldInfos.addAll(subsequentFieldInfos);return fieldInfos;@@ -233,11 +233,11 @@ public class JavaExtractor extends AbstractExtractor {* @before-token: start of possible dims* @after-token: next token after dims, or remain unchanged if no dims*/- private String skipDims(ExtractStream stream) {+ private String skipDims(TokenStream stream) {String dims = "";while (true) {skipAnnotations(stream);- ExtractToken token = stream.current();+ Token token = stream.current();if (token.is(LBRACK)) {dims += "[]";stream.nextClosed(LBRACK, RBRACK);@@ -255,8 +255,8 @@ public class JavaExtractor extends AbstractExtractor {* @before-token: start of type ref* @after-token: next token after type ref*/- private String skipTypeRef(ExtractStream stream) {- ExtractToken token = stream.current();+ private String skipTypeRef(TokenStream stream) {+ Token token = stream.current();String typeRef = token.getText();if (token.is(PRIMITIVES)) {stream.next();@@ -280,14 +280,14 @@ public class JavaExtractor extends AbstractExtractor {* @begin-token: start of a type ref segment including possible annotations* @end-token: next token after type ref segment*/- private String skipTypeRefSegment(ExtractStream stream) {+ private String skipTypeRefSegment(TokenStream stream) {skipModifiers(stream);String identifier = stream.current().getText();- ExtractToken token = stream.next();+ Token token = stream.next();if (token.is(LT)) {int tokenPos = stream.pos();stream.nextClosed(LT, GT);- ExtractStream typeArgStream = new ExtractStream(stream.between(tokenPos, stream.pos()));+ TokenStream typeArgStream = new TokenStream(stream.between(tokenPos, stream.pos()));token = typeArgStream.next();while (true) {skipModifiers(typeArgStream);@@ -309,9 +309,9 @@ public class JavaExtractor extends AbstractExtractor {* @before-token: identifier of the method* @after-token: '}' for class method, ';' for interface method or annotation attribute*/- private void defineMethod(ExtractStream stream, List<Symbol> symbols, TypeDef parent,+ private void defineMethod(TokenStream stream, List<Symbol> symbols, TypeDef parent,List<Modifier> modifiers, @Nullable String typeRef) {- ExtractToken token = stream.current();+ Token token = stream.current();String name = token.getText();TokenPosition pos = token.getPos();String params = null;@@ -386,16 +386,16 @@ public class JavaExtractor extends AbstractExtractor {* @before-token: identifier of field declaration statement* @after-token: end of fields declaration statement, which is ';'*/- private void defineFields(ExtractStream stream, List<Symbol> symbols, TypeDef parent,+ private void defineFields(TokenStream stream, List<Symbol> symbols, TypeDef parent,List<Modifier> modifiers, String typeRef) {- ExtractToken token = stream.current();+ Token token = stream.current();while (!token.is(SEMI)) {stream.next();symbols.add(new FieldDef(parent, token.getText(), token.getPos(), typeRef + skipDims(stream), modifiers));token = stream.current();if (token.is(ASSIGN)) {stream.next();- for (Pair<ExtractToken, String> fieldInfo: skipValue(stream)) {+ for (Pair<Token, String> fieldInfo: skipValue(stream)) {symbols.add(new FieldDef(parent, fieldInfo.getFirst().getText(), fieldInfo.getFirst().getPos(),typeRef + fieldInfo.getSecond(), modifiers));}@@ -412,8 +412,8 @@ public class JavaExtractor extends AbstractExtractor {* @before-token: 'class', 'interface', 'enum', or '@interface'* @after-token: '}'*/- private void defineType(ExtractStream stream, List<Symbol> symbols, Symbol parent, String packageName, List<Modifier> modifiers) {- ExtractToken token = stream.current();+ private void defineType(TokenStream stream, List<Symbol> symbols, Symbol parent, String packageName, List<Modifier> modifiers) {+ Token token = stream.current();if (token.is(AT) && stream.lookAhead(1).is(INTERFACE)) {stream.next().checkType(INTERFACE); // 'interface'stream.next().checkType(Identifier); // identifier@@ -474,11 +474,11 @@ public class JavaExtractor extends AbstractExtractor {* @before-token: type identifier* @after-token: start of type body which is '{'*/- private ExtractToken defineTypeHead(ExtractStream stream) {- ExtractToken typeHeadToken = stream.current();+ private Token defineTypeHead(TokenStream stream) {+ Token typeHeadToken = stream.current();while (true) {- ExtractToken token = stream.nextType(LBRACE, LPAREN);+ Token token = stream.nextType(LBRACE, LPAREN);if (token.is(LPAREN)) {stream.nextClosed(LPAREN, RPAREN);token = stream.next();@@ -496,9 +496,9 @@ public class JavaExtractor extends AbstractExtractor {* @before-token: possible start of modifiers* @after-token: remain unchanged or token after the modifiers if there are modifiers*/- private List<Modifier> skipModifiers(ExtractStream stream) {+ private List<Modifier> skipModifiers(TokenStream stream) {List<Modifier> modifiers = new ArrayList<>();- ExtractToken token = stream.current();+ Token token = stream.current();while (true) {if (token.is(AT) && !stream.lookAhead(1).is(INTERFACE)) {skipAnnotation(stream);@@ -530,8 +530,8 @@ public class JavaExtractor extends AbstractExtractor {* @before-token: possible start of annotations* @after-token: remain unchanged or token after the annotations if there are annotations*/- private void skipAnnotations(ExtractStream stream) {- ExtractToken token = stream.current();+ private void skipAnnotations(TokenStream stream) {+ Token token = stream.current();while (true) {if (token.is(AT) && !stream.lookAhead(1).is(INTERFACE)) {skipAnnotation(stream);@@ -548,10 +548,10 @@ public class JavaExtractor extends AbstractExtractor {* @before-token: '@'* @after-token: token after the annotation*/- private void skipAnnotation(ExtractStream stream) {+ private void skipAnnotation(TokenStream stream) {stream.next();skipTypeName(stream);- ExtractToken token = stream.current();+ Token token = stream.current();if (token.is(LPAREN)) {token = stream.nextClosed(LPAREN, RPAREN);token = stream.next();@@ -564,9 +564,9 @@ public class JavaExtractor extends AbstractExtractor {* @before-token: first section of type name* @after-token: token after type name*/- private String skipTypeName(ExtractStream stream) {+ private String skipTypeName(TokenStream stream) {String typeName = stream.current().getText();- ExtractToken token = stream.next();+ Token token = stream.next();while (token.is(DOT)) {typeName += ".";token = stream.next();diff --git a/commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/ExtractStreamTest.java b/commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/TokenStreamTest.javasimilarity index 85%rename from commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/ExtractStreamTest.javarename to commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/TokenStreamTest.javaindex b1e3d85e78..616b230388 100644--- a/commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/ExtractStreamTest.java+++ b/commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/TokenStreamTest.java@@ -7,11 +7,11 @@ import org.junit.Assert;import org.junit.Test;import com.gitplex.commons.lang.extractors.ExtractException;-import com.gitplex.commons.lang.extractors.ExtractStream;+import com.gitplex.commons.lang.extractors.TokenStream;import com.gitplex.commons.lang.extractors.TokenFilter;import com.gitplex.commons.lang.extractors.java.JavaLexer;-public class ExtractStreamTest {+public class TokenStreamTest {@Testpublic void testBalanced() {@@ -19,7 +19,7 @@ public class ExtractStreamTest {+ " public void sayHello() {System.out.println(\"hello\");}"+ "}";- ExtractStream extractStream = new ExtractStream(+ TokenStream extractStream = new TokenStream(new JavaLexer(new ANTLRInputStream(text)), TokenFilter.DEFAULT_CHANNEL);extractStream.nextType(JavaLexer.LBRACE);@@ -31,7 +31,7 @@ public class ExtractStreamTest {String text = "public class MyClass {"+ " public void sayHello() {System.out.println(\"hello\");}";- ExtractStream extractStream = new ExtractStream(+ TokenStream extractStream = new TokenStream(new JavaLexer(new ANTLRInputStream(text)), TokenFilter.DEFAULT_CHANNEL);extractStream.nextType(JavaLexer.LBRACE);try {