Skip to content

The Unified and Autonomous Development Platform

For clear purpose: rename ExtractToken as Token and rename ExtractStream as TokenStream.

robin shine23524026c686
diff --git a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractToken.java b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/Token.java
similarity index 75%
rename from commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractToken.java
rename to commons.lang/src/main/java/com/gitplex/commons/lang/extractors/Token.java
index 8d9bd4f532..8f6ebc5eee 100644
--- a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractToken.java
+++ b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/Token.java
@@ -2,15 +2,13 @@ package com.gitplex.commons.lang.extractors;
import java.io.Serializable;
-import org.antlr.v4.runtime.Token;
-
import com.gitplex.commons.util.Range;
-public class ExtractToken implements Serializable {
+public class Token implements Serializable {
private static final long serialVersionUID = 1L;
- public static final ExtractToken EOF = new ExtractToken(Token.EOF, "", null);
+ public static final Token EOF = new Token(org.antlr.v4.runtime.Token.EOF, "", null);
private final int type;
@@ -18,7 +16,7 @@ public class ExtractToken implements Serializable {
private final TokenPosition pos;
- public ExtractToken(Token antlrToken) {
+ public Token(org.antlr.v4.runtime.Token antlrToken) {
type = antlrToken.getType();
text = antlrToken.getText();
Range range = new Range(antlrToken.getCharPositionInLine(),
@@ -26,7 +24,7 @@ public class ExtractToken implements Serializable {
pos = new TokenPosition(antlrToken.getLine()-1, range);
}
- public ExtractToken(int type, String text, TokenPosition pos) {
+ public Token(int type, String text, TokenPosition pos) {
this.type = type;
this.text = text;
this.pos = pos;
@@ -49,7 +47,7 @@ public class ExtractToken implements Serializable {
}
public boolean isEof() {
- return type == Token.EOF;
+ return type == org.antlr.v4.runtime.Token.EOF;
}
public int getType() {
@@ -64,13 +62,13 @@ public class ExtractToken implements Serializable {
return pos;
}
- public ExtractToken checkType(int... expectedTypes) {
+ public Token checkType(int... expectedTypes) {
if (!is(expectedTypes))
throw new UnexpectedTokenException(this);
return this;
}
- public ExtractToken checkText(String expectedText) {
+ public Token checkText(String expectedText) {
if (!text.equals(expectedText))
throw new UnexpectedTokenException(this);
return this;
diff --git a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractStream.java b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/TokenStream.java
similarity index 69%
rename from commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractStream.java
rename to commons.lang/src/main/java/com/gitplex/commons/lang/extractors/TokenStream.java
index 8726daa4eb..6a0da55091 100644
--- a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/ExtractStream.java
+++ b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/TokenStream.java
@@ -7,24 +7,23 @@ import org.antlr.v4.runtime.BaseErrorListener;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
-import org.antlr.v4.runtime.Token;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class ExtractStream {
+public class TokenStream {
- private static final Logger logger = LoggerFactory.getLogger(ExtractStream.class);
+ private static final Logger logger = LoggerFactory.getLogger(TokenStream.class);
- private final List<ExtractToken> tokens;
+ private final List<Token> tokens;
private int pos = -1; // 0-indexed position of current token
- public ExtractStream(List<ExtractToken> tokens) {
+ public TokenStream(List<Token> tokens) {
this.tokens = tokens;
- this.tokens.add(ExtractToken.EOF);
+ this.tokens.add(Token.EOF);
}
- public ExtractStream(Lexer lexer, TokenFilter filter) {
+ public TokenStream(Lexer lexer, TokenFilter filter) {
lexer.removeErrorListeners();
lexer.addErrorListener(new BaseErrorListener() {
@@ -36,36 +35,36 @@ public class ExtractStream {
});
tokens = new ArrayList<>();
- Token token = lexer.nextToken();
- while (token.getType() != Token.EOF) {
+ org.antlr.v4.runtime.Token token = lexer.nextToken();
+ while (token.getType() != org.antlr.v4.runtime.Token.EOF) {
if (filter.accept(token))
- tokens.add(new ExtractToken(token));
+ tokens.add(new Token(token));
token = lexer.nextToken();
}
- tokens.add(ExtractToken.EOF);
+ tokens.add(Token.EOF);
}
- public ExtractToken at(int pos) {
+ public Token at(int pos) {
if (pos >=0 && pos < tokens.size())
return tokens.get(pos);
else
throw new ExtractException("Invalid token position: " + pos);
}
- public ExtractToken next() {
+ public Token next() {
return at(++pos);
}
- public ExtractToken lookAhead(int ahead) {
+ public Token lookAhead(int ahead) {
return at(pos+ahead);
}
- public ExtractToken lookBehind(int behind) {
+ public Token lookBehind(int behind) {
return at(pos-behind);
}
- public ExtractToken nextType(int type) {
- ExtractToken token = next();
+ public Token nextType(int type) {
+ Token token = next();
while(true) {
if (token.is(type))
return token;
@@ -73,8 +72,8 @@ public class ExtractStream {
}
}
- public ExtractToken nextType(int...types) {
- ExtractToken token = next();
+ public Token nextType(int...types) {
+ Token token = next();
while(true) {
if (token.is(types))
return token;
@@ -95,9 +94,9 @@ public class ExtractStream {
* @param closeType
* @return
*/
- public ExtractToken nextClosed(int openType, int closeType) {
+ public Token nextClosed(int openType, int closeType) {
int nestingLevel = 1;
- ExtractToken balanced = nextType(openType, closeType);
+ Token balanced = nextType(openType, closeType);
while (true) {
if (balanced.is(closeType)) {
if (--nestingLevel == 0)
@@ -127,14 +126,14 @@ public class ExtractStream {
* @return
* list of tokens between startPos and endPos
*/
- public List<ExtractToken> between(int startPos, int endPos) {
- List<ExtractToken> tokens = new ArrayList<>();
+ public List<Token> between(int startPos, int endPos) {
+ List<Token> tokens = new ArrayList<>();
for (int i=startPos; i<=endPos; i++)
tokens.add(at(i));
return tokens;
}
- public ExtractToken current() {
+ public Token current() {
return at(pos);
}
diff --git a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/UnexpectedTokenException.java b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/UnexpectedTokenException.java
index f2193f5aca..b75e20fb4c 100644
--- a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/UnexpectedTokenException.java
+++ b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/UnexpectedTokenException.java
@@ -4,7 +4,7 @@ package com.gitplex.commons.lang.extractors;
@SuppressWarnings("serial")
public class UnexpectedTokenException extends ExtractException {
- public UnexpectedTokenException(ExtractToken token) {
+ public UnexpectedTokenException(Token token) {
super("Unexpected token: " + token);
}
diff --git a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/java/JavaExtractor.java b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/java/JavaExtractor.java
index d86c9310ec..2b39d195c4 100644
--- a/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/java/JavaExtractor.java
+++ b/commons.lang/src/main/java/com/gitplex/commons/lang/extractors/java/JavaExtractor.java
@@ -11,8 +11,8 @@ import javax.annotation.Nullable;
import org.antlr.v4.runtime.ANTLRInputStream;
import com.gitplex.commons.lang.extractors.AbstractExtractor;
-import com.gitplex.commons.lang.extractors.ExtractStream;
-import com.gitplex.commons.lang.extractors.ExtractToken;
+import com.gitplex.commons.lang.extractors.TokenStream;
+import com.gitplex.commons.lang.extractors.Token;
import com.gitplex.commons.lang.extractors.Symbol;
import com.gitplex.commons.lang.extractors.TokenFilter;
import com.gitplex.commons.lang.extractors.TokenPosition;
@@ -37,12 +37,12 @@ public class JavaExtractor extends AbstractExtractor {
*
*/
public List<Symbol> extract(String text) {
- ExtractStream stream = new ExtractStream(
+ TokenStream stream = new TokenStream(
new JavaLexer(new ANTLRInputStream(text)), TokenFilter.DEFAULT_CHANNEL);
List<Symbol> symbols = new ArrayList<>();
- ExtractToken token = stream.next();
+ Token token = stream.next();
while (token.is(AT) && !stream.lookAhead(1).is(INTERFACE)) {
skipAnnotation(stream);
@@ -88,8 +88,8 @@ public class JavaExtractor extends AbstractExtractor {
* @before-token: '{' or ';' (';' occurs inside of a enum definition)
* @after-token: '}'
*/
- private void defineTypeBody(ExtractStream stream, List<Symbol> symbols, TypeDef parent, String packageName) {
- ExtractToken token = stream.next();
+ private void defineTypeBody(TokenStream stream, List<Symbol> symbols, TypeDef parent, String packageName) {
+ Token token = stream.next();
while(true) {
while (token.is(SEMI))
token = stream.next();
@@ -153,9 +153,9 @@ public class JavaExtractor extends AbstractExtractor {
* fields to be consumed
*
*/
- private List<Pair<ExtractToken, String>> skipValue(ExtractStream stream) {
- List<Pair<ExtractToken, String>> fieldInfos = new ArrayList<>();
- ExtractToken token = stream.current();
+ private List<Pair<Token, String>> skipValue(TokenStream stream) {
+ List<Pair<Token, String>> fieldInfos = new ArrayList<>();
+ Token token = stream.current();
if (token.is(LBRACE)) { // array value
stream.nextClosed(LBRACE, RBRACE);
stream.next();
@@ -171,7 +171,7 @@ public class JavaExtractor extends AbstractExtractor {
break;
} else { // token is ','
token.checkType(COMMA);
- List<Pair<ExtractToken, String>> subsequentFieldTokens = assumeFieldSeparator(stream);
+ List<Pair<Token, String>> subsequentFieldTokens = assumeFieldSeparator(stream);
if (subsequentFieldTokens != null) { // assumption is correct
fieldInfos.addAll(subsequentFieldTokens);
break;
@@ -196,13 +196,13 @@ public class JavaExtractor extends AbstractExtractor {
* null if assumption is incorrect
*
*/
- private List<Pair<ExtractToken, String>> assumeFieldSeparator(ExtractStream stream) {
+ private List<Pair<Token, String>> assumeFieldSeparator(TokenStream stream) {
while (true) {
- ExtractToken token = stream.next();
+ Token token = stream.next();
if (token.is(Identifier)) {
- List<Pair<ExtractToken, String>> fieldInfos = new ArrayList<>();
+ List<Pair<Token, String>> fieldInfos = new ArrayList<>();
stream.next();
- fieldInfos.add(new Pair<ExtractToken, String>(token, skipDims(stream)));
+ fieldInfos.add(new Pair<Token, String>(token, skipDims(stream)));
token = stream.current();
if (token.is(ASSIGN)) { // assumption convinced
stream.next();
@@ -211,7 +211,7 @@ public class JavaExtractor extends AbstractExtractor {
} else if (token.is(SEMI)) { // assumption convinced
return fieldInfos;
} else if (token.is(COMMA)) { // assumption still not convinced
- List<Pair<ExtractToken, String>> subsequentFieldInfos = assumeFieldSeparator(stream);
+ List<Pair<Token, String>> subsequentFieldInfos = assumeFieldSeparator(stream);
if (subsequentFieldInfos != null) {
fieldInfos.addAll(subsequentFieldInfos);
return fieldInfos;
@@ -233,11 +233,11 @@ public class JavaExtractor extends AbstractExtractor {
* @before-token: start of possible dims
* @after-token: next token after dims, or remain unchanged if no dims
*/
- private String skipDims(ExtractStream stream) {
+ private String skipDims(TokenStream stream) {
String dims = "";
while (true) {
skipAnnotations(stream);
- ExtractToken token = stream.current();
+ Token token = stream.current();
if (token.is(LBRACK)) {
dims += "[]";
stream.nextClosed(LBRACK, RBRACK);
@@ -255,8 +255,8 @@ public class JavaExtractor extends AbstractExtractor {
* @before-token: start of type ref
* @after-token: next token after type ref
*/
- private String skipTypeRef(ExtractStream stream) {
- ExtractToken token = stream.current();
+ private String skipTypeRef(TokenStream stream) {
+ Token token = stream.current();
String typeRef = token.getText();
if (token.is(PRIMITIVES)) {
stream.next();
@@ -280,14 +280,14 @@ public class JavaExtractor extends AbstractExtractor {
* @begin-token: start of a type ref segment including possible annotations
* @end-token: next token after type ref segment
*/
- private String skipTypeRefSegment(ExtractStream stream) {
+ private String skipTypeRefSegment(TokenStream stream) {
skipModifiers(stream);
String identifier = stream.current().getText();
- ExtractToken token = stream.next();
+ Token token = stream.next();
if (token.is(LT)) {
int tokenPos = stream.pos();
stream.nextClosed(LT, GT);
- ExtractStream typeArgStream = new ExtractStream(stream.between(tokenPos, stream.pos()));
+ TokenStream typeArgStream = new TokenStream(stream.between(tokenPos, stream.pos()));
token = typeArgStream.next();
while (true) {
skipModifiers(typeArgStream);
@@ -309,9 +309,9 @@ public class JavaExtractor extends AbstractExtractor {
* @before-token: identifier of the method
* @after-token: '}' for class method, ';' for interface method or annotation attribute
*/
- private void defineMethod(ExtractStream stream, List<Symbol> symbols, TypeDef parent,
+ private void defineMethod(TokenStream stream, List<Symbol> symbols, TypeDef parent,
List<Modifier> modifiers, @Nullable String typeRef) {
- ExtractToken token = stream.current();
+ Token token = stream.current();
String name = token.getText();
TokenPosition pos = token.getPos();
String params = null;
@@ -386,16 +386,16 @@ public class JavaExtractor extends AbstractExtractor {
* @before-token: identifier of field declaration statement
* @after-token: end of fields declaration statement, which is ';'
*/
- private void defineFields(ExtractStream stream, List<Symbol> symbols, TypeDef parent,
+ private void defineFields(TokenStream stream, List<Symbol> symbols, TypeDef parent,
List<Modifier> modifiers, String typeRef) {
- ExtractToken token = stream.current();
+ Token token = stream.current();
while (!token.is(SEMI)) {
stream.next();
symbols.add(new FieldDef(parent, token.getText(), token.getPos(), typeRef + skipDims(stream), modifiers));
token = stream.current();
if (token.is(ASSIGN)) {
stream.next();
- for (Pair<ExtractToken, String> fieldInfo: skipValue(stream)) {
+ for (Pair<Token, String> fieldInfo: skipValue(stream)) {
symbols.add(new FieldDef(parent, fieldInfo.getFirst().getText(), fieldInfo.getFirst().getPos(),
typeRef + fieldInfo.getSecond(), modifiers));
}
@@ -412,8 +412,8 @@ public class JavaExtractor extends AbstractExtractor {
* @before-token: 'class', 'interface', 'enum', or '@interface'
* @after-token: '}'
*/
- private void defineType(ExtractStream stream, List<Symbol> symbols, Symbol parent, String packageName, List<Modifier> modifiers) {
- ExtractToken token = stream.current();
+ private void defineType(TokenStream stream, List<Symbol> symbols, Symbol parent, String packageName, List<Modifier> modifiers) {
+ Token token = stream.current();
if (token.is(AT) && stream.lookAhead(1).is(INTERFACE)) {
stream.next().checkType(INTERFACE); // 'interface'
stream.next().checkType(Identifier); // identifier
@@ -474,11 +474,11 @@ public class JavaExtractor extends AbstractExtractor {
* @before-token: type identifier
* @after-token: start of type body which is '{'
*/
- private ExtractToken defineTypeHead(ExtractStream stream) {
- ExtractToken typeHeadToken = stream.current();
+ private Token defineTypeHead(TokenStream stream) {
+ Token typeHeadToken = stream.current();
while (true) {
- ExtractToken token = stream.nextType(LBRACE, LPAREN);
+ Token token = stream.nextType(LBRACE, LPAREN);
if (token.is(LPAREN)) {
stream.nextClosed(LPAREN, RPAREN);
token = stream.next();
@@ -496,9 +496,9 @@ public class JavaExtractor extends AbstractExtractor {
* @before-token: possible start of modifiers
* @after-token: remain unchanged or token after the modifiers if there are modifiers
*/
- private List<Modifier> skipModifiers(ExtractStream stream) {
+ private List<Modifier> skipModifiers(TokenStream stream) {
List<Modifier> modifiers = new ArrayList<>();
- ExtractToken token = stream.current();
+ Token token = stream.current();
while (true) {
if (token.is(AT) && !stream.lookAhead(1).is(INTERFACE)) {
skipAnnotation(stream);
@@ -530,8 +530,8 @@ public class JavaExtractor extends AbstractExtractor {
* @before-token: possible start of annotations
* @after-token: remain unchanged or token after the annotations if there are annotations
*/
- private void skipAnnotations(ExtractStream stream) {
- ExtractToken token = stream.current();
+ private void skipAnnotations(TokenStream stream) {
+ Token token = stream.current();
while (true) {
if (token.is(AT) && !stream.lookAhead(1).is(INTERFACE)) {
skipAnnotation(stream);
@@ -548,10 +548,10 @@ public class JavaExtractor extends AbstractExtractor {
* @before-token: '@'
* @after-token: token after the annotation
*/
- private void skipAnnotation(ExtractStream stream) {
+ private void skipAnnotation(TokenStream stream) {
stream.next();
skipTypeName(stream);
- ExtractToken token = stream.current();
+ Token token = stream.current();
if (token.is(LPAREN)) {
token = stream.nextClosed(LPAREN, RPAREN);
token = stream.next();
@@ -564,9 +564,9 @@ public class JavaExtractor extends AbstractExtractor {
* @before-token: first section of type name
* @after-token: token after type name
*/
- private String skipTypeName(ExtractStream stream) {
+ private String skipTypeName(TokenStream stream) {
String typeName = stream.current().getText();
- ExtractToken token = stream.next();
+ Token token = stream.next();
while (token.is(DOT)) {
typeName += ".";
token = stream.next();
diff --git a/commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/ExtractStreamTest.java b/commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/TokenStreamTest.java
similarity index 85%
rename from commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/ExtractStreamTest.java
rename to commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/TokenStreamTest.java
index b1e3d85e78..616b230388 100644
--- a/commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/ExtractStreamTest.java
+++ b/commons.lang/src/test/java/com/gitplex/commons/lang/extractors/java/TokenStreamTest.java
@@ -7,11 +7,11 @@ import org.junit.Assert;
import org.junit.Test;
import com.gitplex.commons.lang.extractors.ExtractException;
-import com.gitplex.commons.lang.extractors.ExtractStream;
+import com.gitplex.commons.lang.extractors.TokenStream;
import com.gitplex.commons.lang.extractors.TokenFilter;
import com.gitplex.commons.lang.extractors.java.JavaLexer;
-public class ExtractStreamTest {
+public class TokenStreamTest {
@Test
public void testBalanced() {
@@ -19,7 +19,7 @@ public class ExtractStreamTest {
+ " public void sayHello() {System.out.println(\"hello\");}"
+ "}";
- ExtractStream extractStream = new ExtractStream(
+ TokenStream extractStream = new TokenStream(
new JavaLexer(new ANTLRInputStream(text)), TokenFilter.DEFAULT_CHANNEL);
extractStream.nextType(JavaLexer.LBRACE);
@@ -31,7 +31,7 @@ public class ExtractStreamTest {
String text = "public class MyClass {"
+ " public void sayHello() {System.out.println(\"hello\");}";
- ExtractStream extractStream = new ExtractStream(
+ TokenStream extractStream = new TokenStream(
new JavaLexer(new ANTLRInputStream(text)), TokenFilter.DEFAULT_CHANNEL);
extractStream.nextType(JavaLexer.LBRACE);
try {