Skip to content

The Unified and Autonomous Development Platform

Update to use "long" based token to reduce memory usage

robin shen576e36ada53b
diff --git a/core/src/main/java/com/gitplex/server/git/BlobChange.java b/core/src/main/java/com/gitplex/server/git/BlobChange.java
index 90b54dfb46..5557cbcdfe 100644
--- a/core/src/main/java/com/gitplex/server/git/BlobChange.java
+++ b/core/src/main/java/com/gitplex/server/git/BlobChange.java
@@ -11,7 +11,7 @@ import javax.annotation.Nullable;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffEntry.ChangeType;
-import com.gitplex.jsyntax.Token;
+import com.gitplex.jsyntax.Tokenized;
import com.gitplex.server.util.diff.DiffBlock;
import com.gitplex.server.util.diff.DiffUtils;
import com.gitplex.server.util.diff.WhitespaceOption;
@@ -29,7 +29,7 @@ public abstract class BlobChange implements Serializable {
protected final BlobIdent newBlobIdent;
- private transient List<DiffBlock<List<Token>>> diffBlocks;
+ private transient List<DiffBlock<Tokenized>> diffBlocks;
public BlobChange(String oldRev, String newRev, DiffEntry diffEntry,
WhitespaceOption whitespaceOption) {
@@ -82,7 +82,7 @@ public abstract class BlobChange implements Serializable {
return newBlobIdent.path != null? newBlobIdent.path: oldBlobIdent.path;
}
- public List<DiffBlock<List<Token>>> getDiffBlocks() {
+ public List<DiffBlock<Tokenized>> getDiffBlocks() {
if (diffBlocks == null) {
try {
if (type == ChangeType.ADD || type == ChangeType.COPY) {
@@ -140,7 +140,7 @@ public abstract class BlobChange implements Serializable {
public int getAdditions() {
int additions = 0;
- for (DiffBlock<List<Token>> diff: getDiffBlocks()) {
+ for (DiffBlock<Tokenized> diff: getDiffBlocks()) {
if (diff.getOperation() == Operation.INSERT)
additions += diff.getUnits().size();
}
@@ -149,7 +149,7 @@ public abstract class BlobChange implements Serializable {
public int getDeletions() {
int deletions = 0;
- for (DiffBlock<List<Token>> diff: getDiffBlocks()) {
+ for (DiffBlock<Tokenized> diff: getDiffBlocks()) {
if (diff.getOperation() == Operation.DELETE)
deletions += diff.getUnits().size();
}
diff --git a/core/src/main/java/com/gitplex/server/util/StringUtils.java b/core/src/main/java/com/gitplex/server/util/StringUtils.java
index 6dea10d6f8..99e71c56d7 100644
--- a/core/src/main/java/com/gitplex/server/util/StringUtils.java
+++ b/core/src/main/java/com/gitplex/server/util/StringUtils.java
@@ -11,7 +11,7 @@ import java.util.StringTokenizer;
import javax.annotation.Nullable;
-import com.gitplex.jsyntax.TokenizerUtils;
+import com.gitplex.jsyntax.TokenUtils;
import com.google.common.base.CaseFormat;
public class StringUtils extends org.apache.commons.lang3.StringUtils {
@@ -296,7 +296,7 @@ public class StringUtils extends org.apache.commons.lang3.StringUtils {
}
public static String escapeHtml(String text) {
- return TokenizerUtils.escapeHtml(text);
+ return TokenUtils.escapeHtml(text);
}
}
diff --git a/core/src/main/java/com/gitplex/server/util/diff/DiffUtils.java b/core/src/main/java/com/gitplex/server/util/diff/DiffUtils.java
index b6dba7749b..0544ba6df4 100644
--- a/core/src/main/java/com/gitplex/server/util/diff/DiffUtils.java
+++ b/core/src/main/java/com/gitplex/server/util/diff/DiffUtils.java
@@ -10,7 +10,10 @@ import java.util.regex.Pattern;
import javax.annotation.Nullable;
-import com.gitplex.jsyntax.Token;
+import com.gitplex.jsyntax.TextToken;
+import com.gitplex.jsyntax.TokenTypes;
+import com.gitplex.jsyntax.TokenUtils;
+import com.gitplex.jsyntax.Tokenized;
import com.gitplex.jsyntax.Tokenizer;
import com.gitplex.jsyntax.TokenizerRegistry;
import com.gitplex.server.util.StringUtils;
@@ -26,49 +29,54 @@ public class DiffUtils {
private static final Pattern pattern = Pattern.compile("\\w+");
- private static List<String> splitByWord(String line) {
- List<String> tokens = new ArrayList<>();
- Matcher matcher = pattern.matcher(line);
+ private static List<Long> splitByWord(String line, long token) {
+ int beginPos = TokenUtils.getBeginPos(token);
+ int endPos = TokenUtils.getEndPos(token);
+ int typeId = TokenUtils.getTypeId(token);
+ String text = TokenUtils.getText(line, token);
+ List<Long> tokens = new ArrayList<>();
+ Matcher matcher = pattern.matcher(text);
int lastEnd = 0;
while (matcher.find()) {
int start = matcher.start();
if (start > lastEnd)
- tokens.add(line.substring(lastEnd, start));
- tokens.add(matcher.group());
+ tokens.add(TokenUtils.getToken(lastEnd+beginPos, start+beginPos, typeId));
+ tokens.add(TokenUtils.getToken(matcher.start()+beginPos, matcher.end()+beginPos, typeId));
lastEnd = matcher.end();
}
- if (lastEnd < line.length())
- tokens.add(line.substring(lastEnd));
+ if (lastEnd < text.length())
+ tokens.add(TokenUtils.getToken(lastEnd+beginPos, endPos, typeId));
return tokens;
}
- private static List<List<Token>> tokenize(List<String> lines, @Nullable String fileName) {
+ private static List<Tokenized> tokenize(List<String> lines, @Nullable String fileName) {
Tokenizer tokenizer = TokenizerRegistry.getTokenizer(fileName);
if (tokenizer != null) {
- List<List<Token>> tokenizedLines = tokenizer.tokenize(lines);
- List<List<Token>> refinedLines = new ArrayList<>();
- for (List<Token> tokenizedLine: tokenizedLines) {
- List<Token> refinedLine = new ArrayList<>();
- for (Token token: tokenizedLine) {
- if (token.getType().equals("") || token.isComment() || token.isString()
- || token.isMeta() || token.isLink() || token.isAttribute()
- || token.isProperty()) {
- for (String each: splitByWord(token.getText()))
- refinedLine.add(new Token(token.getType(), each));
+ List<Tokenized> tokenizedLines = tokenizer.tokenize(lines);
+ List<Tokenized> refinedTokens = new ArrayList<>();
+ int index = 0;
+ for (Tokenized tokenizedLine: tokenizedLines) {
+ List<Long> refinedLine = new ArrayList<>();
+ for (long token: tokenizedLine.getTokens()) {
+ int typeId = TokenUtils.getTypeId(token);
+ if (typeId == 0 || (typeId & TokenTypes.COMMENT) != 0 || (typeId & TokenTypes.STRING) != 0
+ || (typeId & TokenTypes.STRING2) != 0 || (typeId & TokenTypes.META) != 0
+ || (typeId & TokenTypes.LINK) != 0 || (typeId & TokenTypes.ATTRIBUTE) != 0
+ || (typeId & TokenTypes.PROPERTY) != 0) {
+ refinedLine.addAll(splitByWord(lines.get(index), token));
} else {
refinedLine.add(token);
}
}
- refinedLines.add(refinedLine);
+ refinedTokens.add(new Tokenized(tokenizedLine.getText(), TokenUtils.toArray(refinedLine)));
+ index++;
}
- return refinedLines;
+ return refinedTokens;
} else {
- List<List<Token>> tokenizedLines = new ArrayList<>();
+ List<Tokenized> tokenizedLines = new ArrayList<>();
for (String line: lines) {
- List<Token> tokenizedLine = new ArrayList<>();
- for (String each: splitByWord(line))
- tokenizedLine.add(new Token("", each));
- tokenizedLines.add(tokenizedLine);
+ long token = TokenUtils.getToken(0, line.length(), 0);
+ tokenizedLines.add(new Tokenized(line, TokenUtils.toArray(splitByWord(line, token))));
}
return tokenizedLines;
}
@@ -77,7 +85,7 @@ public class DiffUtils {
/**
* Diff two list of strings.
*/
- public static List<DiffBlock<List<Token>>> diff(List<String> oldLines, @Nullable String oldFileName,
+ public static List<DiffBlock<Tokenized>> diff(List<String> oldLines, @Nullable String oldFileName,
List<String> newLines, @Nullable String newFileName, WhitespaceOption whitespaceOption) {
Preconditions.checkArgument(oldLines.size() + newLines.size() <= MAX_DIFF_SIZE,
"Total size of old lines and new lines should be less than " + MAX_DIFF_SIZE + ".");
@@ -92,36 +100,36 @@ public class DiffUtils {
processedNewLines.add(whitespaceOption.process(line));
}
- List<List<Token>> oldTokenizedLines = tokenize(oldLines, oldFileName);
- List<List<Token>> newTokenizedLines = tokenize(newLines, newFileName);
+ List<Tokenized> oldTokenizedLines = tokenize(oldLines, oldFileName);
+ List<Tokenized> newTokenizedLines = tokenize(newLines, newFileName);
DiffMatchPatch dmp = new DiffMatchPatch();
TokensToCharsResult<String> result1 = tokensToChars(processedOldLines, processedNewLines);
List<DiffMatchPatch.Diff> diffs = dmp.diff_main(result1.chars1, result1.chars2, false);
- List<DiffBlock<List<Token>>> diffBlocks = new ArrayList<>();
+ List<DiffBlock<Tokenized>> diffBlocks = new ArrayList<>();
int oldLineNo = 0;
int newLineNo = 0;
for (Diff diff : diffs) {
- List<List<Token>> lines = new ArrayList<>();
+ List<Tokenized> lines = new ArrayList<>();
if (diff.operation == Operation.EQUAL) {
for (int i = 0; i < diff.text.length(); i++) {
lines.add(newTokenizedLines.get(newLineNo));
oldLineNo++;
newLineNo++;
}
- diffBlocks.add(new DiffBlock<List<Token>>(diff.operation, lines,
+ diffBlocks.add(new DiffBlock<>(diff.operation, lines,
oldLineNo-lines.size(), newLineNo-lines.size()));
} else if (diff.operation == Operation.INSERT) {
for (int i = 0; i < diff.text.length(); i++)
lines.add(newTokenizedLines.get(newLineNo++));
- diffBlocks.add(new DiffBlock<List<Token>>(diff.operation, lines,
+ diffBlocks.add(new DiffBlock<>(diff.operation, lines,
oldLineNo, newLineNo-lines.size()));
} else {
for (int i = 0; i < diff.text.length(); i++)
lines.add(oldTokenizedLines.get(oldLineNo++));
- diffBlocks.add(new DiffBlock<List<Token>>(diff.operation, lines,
+ diffBlocks.add(new DiffBlock<>(diff.operation, lines,
oldLineNo-lines.size(), newLineNo));
}
}
@@ -173,7 +181,7 @@ public class DiffUtils {
* @return
*/
public static LinkedHashMap<Integer, LineDiff> align(
- List<List<Token>> deleteLines, List<List<Token>> insertLines) {
+ List<Tokenized> deleteLines, List<Tokenized> insertLines) {
LinkedHashMap<Integer, LineDiff> lineDiffs = new LinkedHashMap<>();
DiffMatchPatch dmp = new DiffMatchPatch();
@@ -181,17 +189,20 @@ public class DiffUtils {
long time = System.currentTimeMillis();
int nextInsert = 0;
for (int i=0; i<deleteLines.size(); i++) {
- List<Token> deleteLine = deleteLines.get(i);
+ Tokenized deleteLine = deleteLines.get(i);
+ List<TextToken> deleteTokens = TokenUtils.getTextTokens(deleteLine);
for (int j=nextInsert; j<insertLines.size(); j++) {
- List<Token> insertLine = insertLines.get(j);
- TokensToCharsResult<Token> result = DiffUtils.tokensToChars(deleteLine, insertLine);
+ Tokenized insertLine = insertLines.get(j);
+ List<TextToken> insertTokens = TokenUtils.getTextTokens(insertLine);
+
+ TokensToCharsResult<TextToken> result = DiffUtils.tokensToChars(deleteTokens, insertTokens);
List<DiffMatchPatch.Diff> diffs = dmp.diff_main(result.chars1, result.chars2, false);
int equal = 0;
int total = 0;
for (DiffMatchPatch.Diff diff: diffs) {
for (int k=0; k<diff.text.length(); k++) {
int pos = diff.text.charAt(k);
- Token token = result.tokenArray.get(pos);
+ TextToken token = result.tokenArray.get(pos);
if (StringUtils.isNotBlank(token.getText())) {
total += token.getText().length();
if (diff.operation == Operation.EQUAL)
@@ -200,26 +211,29 @@ public class DiffUtils {
}
}
if (equal*3 >= total) {
- List<DiffBlock<Token>> diffBlocks = new ArrayList<>();
+ List<DiffBlock<TextToken>> diffBlocks = new ArrayList<>();
int oldLineNo = 0;
int newLineNo = 0;
for (Diff diff : diffs) {
- List<Token> tokens = new ArrayList<>();
+ List<TextToken> tokens = new ArrayList<>();
if (diff.operation == Operation.EQUAL) {
for (int k = 0; k < diff.text.length(); k++) {
- tokens.add(insertLine.get(newLineNo));
+ tokens.add(insertTokens.get(newLineNo));
oldLineNo++;
newLineNo++;
}
- diffBlocks.add(new DiffBlock<Token>(diff.operation, tokens, oldLineNo-tokens.size(), newLineNo-tokens.size()));
+ diffBlocks.add(new DiffBlock<TextToken>(
+ diff.operation, tokens, oldLineNo-tokens.size(), newLineNo-tokens.size()));
} else if (diff.operation == Operation.INSERT) {
for (int k = 0; k < diff.text.length(); k++)
- tokens.add(insertLine.get(newLineNo++));
- diffBlocks.add(new DiffBlock<Token>(diff.operation, tokens, oldLineNo, newLineNo-tokens.size()));
+ tokens.add(insertTokens.get(newLineNo++));
+ diffBlocks.add(new DiffBlock<>(
+ diff.operation, tokens, oldLineNo, newLineNo-tokens.size()));
} else {
for (int k = 0; k < diff.text.length(); k++)
- tokens.add(deleteLine.get(oldLineNo++));
- diffBlocks.add(new DiffBlock<Token>(diff.operation, tokens, oldLineNo-tokens.size(), newLineNo));
+ tokens.add(deleteTokens.get(oldLineNo++));
+ diffBlocks.add(new DiffBlock<>(
+ diff.operation, tokens, oldLineNo-tokens.size(), newLineNo));
}
}
diff --git a/core/src/main/java/com/gitplex/server/util/diff/LineDiff.java b/core/src/main/java/com/gitplex/server/util/diff/LineDiff.java
index a63238e69c..384a741905 100644
--- a/core/src/main/java/com/gitplex/server/util/diff/LineDiff.java
+++ b/core/src/main/java/com/gitplex/server/util/diff/LineDiff.java
@@ -3,7 +3,7 @@ package com.gitplex.server.util.diff;
import java.io.Serializable;
import java.util.List;
-import com.gitplex.jsyntax.Token;
+import com.gitplex.jsyntax.TextToken;
public class LineDiff implements Serializable {
@@ -11,18 +11,18 @@ public class LineDiff implements Serializable {
private final int compareLine;
- private final List<DiffBlock<Token>> tokenDiffs;
+ private final List<DiffBlock<TextToken>> tokenDiffs;
- public LineDiff(int line, List<DiffBlock<Token>> diffs) {
- this.compareLine = line;
- this.tokenDiffs = diffs;
+ public LineDiff(int compareLine, List<DiffBlock<TextToken>> tokenDiffs) {
+ this.compareLine = compareLine;
+ this.tokenDiffs = tokenDiffs;
}
public int getCompareLine() {
return compareLine;
}
- public List<DiffBlock<Token>> getTokenDiffs() {
+ public List<DiffBlock<TextToken>> getTokenDiffs() {
return tokenDiffs;
}
diff --git a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/MarkAwareDiffBlock.java b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/MarkAwareDiffBlock.java
index 6622311d80..82efbf738b 100644
--- a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/MarkAwareDiffBlock.java
+++ b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/MarkAwareDiffBlock.java
@@ -3,7 +3,7 @@ package com.gitplex.server.web.component.diff.blob.text;
import java.io.Serializable;
import java.util.List;
-import com.gitplex.jsyntax.Token;
+import com.gitplex.jsyntax.Tokenized;
public class MarkAwareDiffBlock implements Serializable {
@@ -15,11 +15,11 @@ public class MarkAwareDiffBlock implements Serializable {
private final Type type;
- private final List<List<Token>> lines;
+ private final List<Tokenized> lines;
private final int oldStart, newStart;
- public MarkAwareDiffBlock(Type type, List<List<Token>> lines, int oldStart, int newStart) {
+ public MarkAwareDiffBlock(Type type, List<Tokenized> lines, int oldStart, int newStart) {
this.type = type;
this.lines = lines;
this.oldStart = oldStart;
@@ -30,7 +30,7 @@ public class MarkAwareDiffBlock implements Serializable {
return type;
}
- public List<List<Token>> getLines() {
+ public List<Tokenized> getLines() {
return lines;
}
diff --git a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java
index 4c03464c49..7917f34132 100644
--- a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java
+++ b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java
@@ -37,7 +37,9 @@ import org.unbescape.html.HtmlEscape;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.gitplex.jsymbol.Range;
-import com.gitplex.jsyntax.Token;
+import com.gitplex.jsyntax.TextToken;
+import com.gitplex.jsyntax.TokenUtils;
+import com.gitplex.jsyntax.Tokenized;
import com.gitplex.server.GitPlex;
import com.gitplex.server.git.Blame;
import com.gitplex.server.git.BlobChange;
@@ -624,7 +626,8 @@ public class TextDiffPanel extends Panel implements SourceAware {
appendDeletesAndInserts(builder, block, nextBlock, prevDeleteLineIndex, deleteLineIndex,
prevInsertLineIndex, insertLineIndex);
- appendModification(builder, block, nextBlock, deleteLineIndex, insertLineIndex, lineChange.getTokenDiffs());
+ appendModification(builder, block, nextBlock, deleteLineIndex, insertLineIndex,
+ lineChange.getTokenDiffs());
prevDeleteLineIndex = deleteLineIndex+1;
prevInsertLineIndex = insertLineIndex+1;
@@ -721,6 +724,16 @@ public class TextDiffPanel extends Panel implements SourceAware {
return null;
}
+ private void appendTokenized(StringBuilder builder, Tokenized tokenized) {
+ if (tokenized.getTokens().length == 0) {
+ builder.append("&nbsp;");
+ } else {
+ for (long token: tokenized.getTokens()) {
+ builder.append(TokenUtils.toHtml(tokenized.getText(), token, null));
+ }
+ }
+ }
+
private void appendEqual(StringBuilder builder, MarkAwareDiffBlock block, int lineIndex, int lastContextSize) {
if (lastContextSize != 0)
builder.append("<tr class='code expanded'>");
@@ -738,14 +751,7 @@ public class TextDiffPanel extends Panel implements SourceAware {
builder.append("<td class='number noselect'>").append(newLineNo+1).append("</td>");
builder.append("<td class='operation noselect'>&nbsp;</td>");
builder.append("<td class='content' data-old='").append(oldLineNo).append("' data-new='").append(newLineNo).append("'>");
- List<Token> tokens = block.getLines().get(lineIndex);
- if (tokens.isEmpty()) {
- builder.append("&nbsp;");
- } else {
- for (Token token: tokens) {
- builder.append(token.toHtml(null));
- }
- }
+ appendTokenized(builder, block.getLines().get(lineIndex));
builder.append("</td>");
} else {
if (blameInfo != null) {
@@ -754,14 +760,7 @@ public class TextDiffPanel extends Panel implements SourceAware {
builder.append("<td class='number noselect'>").append(oldLineNo+1).append("</td>");
builder.append("<td class='operation noselect'>&nbsp;</td>");
builder.append("<td class='content left' data-old='").append(oldLineNo).append("' data-new='").append(newLineNo).append("'>");
- List<Token> tokens = block.getLines().get(lineIndex);
- if (tokens.isEmpty()) {
- builder.append("&nbsp;");
- } else {
- for (Token token: tokens) {
- builder.append(token.toHtml(null));
- }
- }
+ appendTokenized(builder, block.getLines().get(lineIndex));
builder.append("</td>");
if (blameInfo != null) {
appendBlame(builder, -1, newLineNo);
@@ -769,20 +768,14 @@ public class TextDiffPanel extends Panel implements SourceAware {
builder.append("<td class='number noselect'>").append(newLineNo+1).append("</td>");
builder.append("<td class='operation noselect'>&nbsp;</td>");
builder.append("<td class='content right' data-old='").append(oldLineNo).append("' data-new='").append(newLineNo).append("'>");
- tokens = block.getLines().get(lineIndex);
- if (tokens.isEmpty()) {
- builder.append("&nbsp;");
- } else {
- for (Token token: tokens) {
- builder.append(token.toHtml(null));
- }
- }
+ appendTokenized(builder, block.getLines().get(lineIndex));
builder.append("</td>");
}
builder.append("</tr>");
}
- private void appendInsert(StringBuilder builder, MarkAwareDiffBlock block, int lineIndex, @Nullable List<DiffBlock<Token>> tokenDiffs) {
+ private void appendInsert(StringBuilder builder, MarkAwareDiffBlock block, int lineIndex,
+ @Nullable List<DiffBlock<TextToken>> tokenDiffs) {
builder.append("<tr class='code original'>");
int newLineNo = block.getNewStart() + lineIndex;
@@ -798,21 +791,15 @@ public class TextDiffPanel extends Panel implements SourceAware {
if (tokenDiffs.isEmpty()) {
builder.append("&nbsp;");
} else {
- for (DiffBlock<Token> tokenBlock: tokenDiffs) {
- for (Token token: tokenBlock.getUnits()) {
+ for (DiffBlock<TextToken> tokenBlock: tokenDiffs) {
+ for (TextToken token: tokenBlock.getUnits()) {
if (tokenBlock.getOperation() != Operation.DELETE)
- builder.append(token.toHtml(getOperationClass(tokenBlock.getOperation())));
+ builder.append(TokenUtils.toHtml(token, getOperationClass(tokenBlock.getOperation())));
}
}
}
} else {
- List<Token> tokens = block.getLines().get(lineIndex);
- if (tokens.isEmpty()) {
- builder.append("&nbsp;");
- } else {
- for (int i=0; i<tokens.size(); i++)
- builder.append(tokens.get(i).toHtml(null));
- }
+ appendTokenized(builder, block.getLines().get(lineIndex));
}
builder.append("</td>");
} else {
@@ -828,19 +815,14 @@ public class TextDiffPanel extends Panel implements SourceAware {
builder.append("<td class='number noselect new'>").append(newLineNo+1).append("</td>");
builder.append("<td class='operation noselect new'>+</td>");
builder.append("<td class='content right new' data-new='").append(newLineNo).append("'>");
- List<Token> tokens = block.getLines().get(lineIndex);
- if (tokens.isEmpty()) {
- builder.append("&nbsp;");
- } else {
- for (int i=0; i<tokens.size(); i++)
- builder.append(tokens.get(i).toHtml(null));
- }
+ appendTokenized(builder, block.getLines().get(lineIndex));
builder.append("</td>");
}
builder.append("</tr>");
}
- private void appendDelete(StringBuilder builder, MarkAwareDiffBlock block, int lineIndex, @Nullable List<DiffBlock<Token>> tokenDiffs) {
+ private void appendDelete(StringBuilder builder, MarkAwareDiffBlock block, int lineIndex,
+ @Nullable List<DiffBlock<TextToken>> tokenDiffs) {
builder.append("<tr class='code original'>");
int oldLineNo = block.getOldStart() + lineIndex;
@@ -856,21 +838,15 @@ public class TextDiffPanel extends Panel implements SourceAware {
if (tokenDiffs.isEmpty()) {
builder.append("&nbsp;");
} else {
- for (DiffBlock<Token> tokenBlock: tokenDiffs) {
- for (Token token: tokenBlock.getUnits()) {
+ for (DiffBlock<TextToken> tokenBlock: tokenDiffs) {
+ for (TextToken token: tokenBlock.getUnits()) {
if (tokenBlock.getOperation() != Operation.INSERT)
- builder.append(token.toHtml(getOperationClass(tokenBlock.getOperation())));
+ builder.append(TokenUtils.toHtml(token, getOperationClass(tokenBlock.getOperation())));
}
}
}
} else {
- List<Token> tokens = block.getLines().get(lineIndex);
- if (tokens.isEmpty()) {
- builder.append("&nbsp;");
- } else {
- for (int i=0; i<tokens.size(); i++)
- builder.append(tokens.get(i).toHtml(null));
- }
+ appendTokenized(builder, block.getLines().get(lineIndex));
}
builder.append("</td>");
} else {
@@ -880,13 +856,7 @@ public class TextDiffPanel extends Panel implements SourceAware {
builder.append("<td class='number noselect old'>").append(oldLineNo+1).append("</td>");
builder.append("<td class='operation noselect old'>-</td>");
builder.append("<td class='content left old' data-old='").append(oldLineNo).append("'>");
- List<Token> tokens = block.getLines().get(lineIndex);
- if (tokens.isEmpty()) {
- builder.append("&nbsp;");
- } else {
- for (int i=0; i<tokens.size(); i++)
- builder.append(tokens.get(i).toHtml(null));
- }
+ appendTokenized(builder, block.getLines().get(lineIndex));
builder.append("</td>");
if (blameInfo != null) {
builder.append("<td class='blame noselect'>&nbsp;</td>");
@@ -909,13 +879,7 @@ public class TextDiffPanel extends Panel implements SourceAware {
builder.append("<td class='number noselect old'>").append(oldLineNo+1).append("</td>");
builder.append("<td class='operation noselect old'>-</td>");
builder.append("<td class='content left old' data-old='").append(oldLineNo).append("'>");
- List<Token> tokens = deleteBlock.getLines().get(deleteLineIndex);
- if (tokens.isEmpty()) {
- builder.append("&nbsp;");
- } else {
- for (Token token: tokens)
- builder.append(token.toHtml(null));
- }
+ appendTokenized(builder, deleteBlock.getLines().get(deleteLineIndex));
builder.append("</td>");
int newLineNo = insertBlock.getNewStart()+insertLineIndex;
@@ -925,13 +889,7 @@ public class TextDiffPanel extends Panel implements SourceAware {
builder.append("<td class='number noselect new'>").append(newLineNo+1).append("</td>");
builder.append("<td class='operation noselect new'>+</td>");
builder.append("<td class='content right new' data-new='").append(newLineNo).append("'>");
- tokens = insertBlock.getLines().get(insertLineIndex);
- if (tokens.isEmpty()) {
- builder.append("&nbsp;");
- } else {
- for (Token token: tokens)
- builder.append(token.toHtml(null));
- }
+ appendTokenized(builder, insertBlock.getLines().get(insertLineIndex));
builder.append("</td>");
builder.append("</tr>");
@@ -939,7 +897,7 @@ public class TextDiffPanel extends Panel implements SourceAware {
private void appendModification(StringBuilder builder, MarkAwareDiffBlock deleteBlock,
MarkAwareDiffBlock insertBlock, int deleteLineIndex, int insertLineIndex,
- List<DiffBlock<Token>> tokenDiffs) {
+ List<DiffBlock<TextToken>> tokenDiffs) {
builder.append("<tr class='code original'>");
int oldLineNo = deleteBlock.getOldStart() + deleteLineIndex;
@@ -955,9 +913,9 @@ public class TextDiffPanel extends Panel implements SourceAware {
if (tokenDiffs.isEmpty()) {
builder.append("&nbsp;");
} else {
- for (DiffBlock<Token> tokenBlock: tokenDiffs) {
- for (Token token: tokenBlock.getUnits())
- builder.append(token.toHtml(getOperationClass(tokenBlock.getOperation())));
+ for (DiffBlock<TextToken> tokenBlock: tokenDiffs) {
+ for (TextToken token: tokenBlock.getUnits())
+ builder.append(TokenUtils.toHtml(token, getOperationClass(tokenBlock.getOperation())));
}
}
builder.append("</td>");
@@ -971,10 +929,10 @@ public class TextDiffPanel extends Panel implements SourceAware {
if (tokenDiffs.isEmpty()) {
builder.append("&nbsp;");
} else {
- for (DiffBlock<Token> tokenBlock: tokenDiffs) {
- for (Token token: tokenBlock.getUnits()) {
+ for (DiffBlock<TextToken> tokenBlock: tokenDiffs) {
+ for (TextToken token: tokenBlock.getUnits()) {
if (tokenBlock.getOperation() != Operation.INSERT)
- builder.append(token.toHtml(getOperationClass(tokenBlock.getOperation())));
+ builder.append(TokenUtils.toHtml(token, getOperationClass(tokenBlock.getOperation())));
}
}
}
@@ -989,10 +947,10 @@ public class TextDiffPanel extends Panel implements SourceAware {
if (tokenDiffs.isEmpty()) {
builder.append("&nbsp;");
} else {
- for (DiffBlock<Token> tokenBlock: tokenDiffs) {
- for (Token token: tokenBlock.getUnits()) {
+ for (DiffBlock<TextToken> tokenBlock: tokenDiffs) {
+ for (TextToken token: tokenBlock.getUnits()) {
if (tokenBlock.getOperation() != Operation.DELETE)
- builder.append(token.toHtml(getOperationClass(tokenBlock.getOperation())));
+ builder.append(TokenUtils.toHtml(token, getOperationClass(tokenBlock.getOperation())));
}
}
}
@@ -1064,7 +1022,7 @@ public class TextDiffPanel extends Panel implements SourceAware {
}
}
- for (DiffBlock<List<Token>> diffBlock: change.getDiffBlocks()) {
+ for (DiffBlock<Tokenized> diffBlock: change.getDiffBlocks()) {
if (diffBlock.getOperation() == Operation.DELETE) {
diffBlocks.add(new MarkAwareDiffBlock(Type.DELETE, diffBlock.getUnits(),
diffBlock.getOldStart(), diffBlock.getNewStart()));
@@ -1091,12 +1049,12 @@ public class TextDiffPanel extends Panel implements SourceAware {
if (to > diffBlock.getUnits().size())
to = diffBlock.getUnits().size();
if (lastIndex < from) {
- List<List<Token>> lines = diffBlock.getUnits().subList(lastIndex, from);
+ List<Tokenized> lines = diffBlock.getUnits().subList(lastIndex, from);
diffBlocks.add(new MarkAwareDiffBlock(Type.EQUAL, lines,
diffBlock.getOldStart()+lastIndex, diffBlock.getNewStart()+lastIndex));
}
if (from < to) {
- List<List<Token>> lines = diffBlock.getUnits().subList(from, to);
+ List<Tokenized> lines = diffBlock.getUnits().subList(from, to);
diffBlocks.add(new MarkAwareDiffBlock(Type.MARKED_EQUAL, lines,
diffBlock.getOldStart()+from, diffBlock.getNewStart()+from));
}
@@ -1105,7 +1063,7 @@ public class TextDiffPanel extends Panel implements SourceAware {
}
if (lastIndex < diffBlock.getUnits().size()) {
- List<List<Token>> lines = diffBlock.getUnits().subList(lastIndex, diffBlock.getUnits().size());
+ List<Tokenized> lines = diffBlock.getUnits().subList(lastIndex, diffBlock.getUnits().size());
diffBlocks.add(new MarkAwareDiffBlock(Type.EQUAL, lines,
diffBlock.getOldStart()+lastIndex, diffBlock.getNewStart()+lastIndex));
}