The Unified and Autonomous Development Platform
Update to use "long" based token to reduce memory usage
robin shen576e36ada53b
diff --git a/core/src/main/java/com/gitplex/server/git/BlobChange.java b/core/src/main/java/com/gitplex/server/git/BlobChange.javaindex 90b54dfb46..5557cbcdfe 100644--- a/core/src/main/java/com/gitplex/server/git/BlobChange.java+++ b/core/src/main/java/com/gitplex/server/git/BlobChange.java@@ -11,7 +11,7 @@ import javax.annotation.Nullable;import org.eclipse.jgit.diff.DiffEntry;import org.eclipse.jgit.diff.DiffEntry.ChangeType;-import com.gitplex.jsyntax.Token;+import com.gitplex.jsyntax.Tokenized;import com.gitplex.server.util.diff.DiffBlock;import com.gitplex.server.util.diff.DiffUtils;import com.gitplex.server.util.diff.WhitespaceOption;@@ -29,7 +29,7 @@ public abstract class BlobChange implements Serializable {protected final BlobIdent newBlobIdent;- private transient List<DiffBlock<List<Token>>> diffBlocks;+ private transient List<DiffBlock<Tokenized>> diffBlocks;public BlobChange(String oldRev, String newRev, DiffEntry diffEntry,WhitespaceOption whitespaceOption) {@@ -82,7 +82,7 @@ public abstract class BlobChange implements Serializable {return newBlobIdent.path != null? newBlobIdent.path: oldBlobIdent.path;}- public List<DiffBlock<List<Token>>> getDiffBlocks() {+ public List<DiffBlock<Tokenized>> getDiffBlocks() {if (diffBlocks == null) {try {if (type == ChangeType.ADD || type == ChangeType.COPY) {@@ -140,7 +140,7 @@ public abstract class BlobChange implements Serializable {public int getAdditions() {int additions = 0;- for (DiffBlock<List<Token>> diff: getDiffBlocks()) {+ for (DiffBlock<Tokenized> diff: getDiffBlocks()) {if (diff.getOperation() == Operation.INSERT)additions += diff.getUnits().size();}@@ -149,7 +149,7 @@ public abstract class BlobChange implements Serializable {public int getDeletions() {int deletions = 0;- for (DiffBlock<List<Token>> diff: getDiffBlocks()) {+ for (DiffBlock<Tokenized> diff: getDiffBlocks()) {if (diff.getOperation() == Operation.DELETE)deletions += diff.getUnits().size();}diff --git a/core/src/main/java/com/gitplex/server/util/StringUtils.java b/core/src/main/java/com/gitplex/server/util/StringUtils.javaindex 6dea10d6f8..99e71c56d7 100644--- a/core/src/main/java/com/gitplex/server/util/StringUtils.java+++ b/core/src/main/java/com/gitplex/server/util/StringUtils.java@@ -11,7 +11,7 @@ import java.util.StringTokenizer;import javax.annotation.Nullable;-import com.gitplex.jsyntax.TokenizerUtils;+import com.gitplex.jsyntax.TokenUtils;import com.google.common.base.CaseFormat;public class StringUtils extends org.apache.commons.lang3.StringUtils {@@ -296,7 +296,7 @@ public class StringUtils extends org.apache.commons.lang3.StringUtils {}public static String escapeHtml(String text) {- return TokenizerUtils.escapeHtml(text);+ return TokenUtils.escapeHtml(text);}}diff --git a/core/src/main/java/com/gitplex/server/util/diff/DiffUtils.java b/core/src/main/java/com/gitplex/server/util/diff/DiffUtils.javaindex b6dba7749b..0544ba6df4 100644--- a/core/src/main/java/com/gitplex/server/util/diff/DiffUtils.java+++ b/core/src/main/java/com/gitplex/server/util/diff/DiffUtils.java@@ -10,7 +10,10 @@ import java.util.regex.Pattern;import javax.annotation.Nullable;-import com.gitplex.jsyntax.Token;+import com.gitplex.jsyntax.TextToken;+import com.gitplex.jsyntax.TokenTypes;+import com.gitplex.jsyntax.TokenUtils;+import com.gitplex.jsyntax.Tokenized;import com.gitplex.jsyntax.Tokenizer;import com.gitplex.jsyntax.TokenizerRegistry;import com.gitplex.server.util.StringUtils;@@ -26,49 +29,54 @@ public class DiffUtils {private static final Pattern pattern = Pattern.compile("\\w+");- private static List<String> splitByWord(String line) {- List<String> tokens = new ArrayList<>();- Matcher matcher = pattern.matcher(line);+ private static List<Long> splitByWord(String line, long token) {+ int beginPos = TokenUtils.getBeginPos(token);+ int endPos = TokenUtils.getEndPos(token);+ int typeId = TokenUtils.getTypeId(token);+ String text = TokenUtils.getText(line, token);+ List<Long> tokens = new ArrayList<>();+ Matcher matcher = pattern.matcher(text);int lastEnd = 0;while (matcher.find()) {int start = matcher.start();if (start > lastEnd)- tokens.add(line.substring(lastEnd, start));- tokens.add(matcher.group());+ tokens.add(TokenUtils.getToken(lastEnd+beginPos, start+beginPos, typeId));+ tokens.add(TokenUtils.getToken(matcher.start()+beginPos, matcher.end()+beginPos, typeId));lastEnd = matcher.end();}- if (lastEnd < line.length())- tokens.add(line.substring(lastEnd));+ if (lastEnd < text.length())+ tokens.add(TokenUtils.getToken(lastEnd+beginPos, endPos, typeId));return tokens;}- private static List<List<Token>> tokenize(List<String> lines, @Nullable String fileName) {+ private static List<Tokenized> tokenize(List<String> lines, @Nullable String fileName) {Tokenizer tokenizer = TokenizerRegistry.getTokenizer(fileName);if (tokenizer != null) {- List<List<Token>> tokenizedLines = tokenizer.tokenize(lines);- List<List<Token>> refinedLines = new ArrayList<>();- for (List<Token> tokenizedLine: tokenizedLines) {- List<Token> refinedLine = new ArrayList<>();- for (Token token: tokenizedLine) {- if (token.getType().equals("") || token.isComment() || token.isString()- || token.isMeta() || token.isLink() || token.isAttribute()- || token.isProperty()) {- for (String each: splitByWord(token.getText()))- refinedLine.add(new Token(token.getType(), each));+ List<Tokenized> tokenizedLines = tokenizer.tokenize(lines);+ List<Tokenized> refinedTokens = new ArrayList<>();+ int index = 0;+ for (Tokenized tokenizedLine: tokenizedLines) {+ List<Long> refinedLine = new ArrayList<>();+ for (long token: tokenizedLine.getTokens()) {+ int typeId = TokenUtils.getTypeId(token);+ if (typeId == 0 || (typeId & TokenTypes.COMMENT) != 0 || (typeId & TokenTypes.STRING) != 0+ || (typeId & TokenTypes.STRING2) != 0 || (typeId & TokenTypes.META) != 0+ || (typeId & TokenTypes.LINK) != 0 || (typeId & TokenTypes.ATTRIBUTE) != 0+ || (typeId & TokenTypes.PROPERTY) != 0) {+ refinedLine.addAll(splitByWord(lines.get(index), token));} else {refinedLine.add(token);}}- refinedLines.add(refinedLine);+ refinedTokens.add(new Tokenized(tokenizedLine.getText(), TokenUtils.toArray(refinedLine)));+ index++;}- return refinedLines;+ return refinedTokens;} else {- List<List<Token>> tokenizedLines = new ArrayList<>();+ List<Tokenized> tokenizedLines = new ArrayList<>();for (String line: lines) {- List<Token> tokenizedLine = new ArrayList<>();- for (String each: splitByWord(line))- tokenizedLine.add(new Token("", each));- tokenizedLines.add(tokenizedLine);+ long token = TokenUtils.getToken(0, line.length(), 0);+ tokenizedLines.add(new Tokenized(line, TokenUtils.toArray(splitByWord(line, token))));}return tokenizedLines;}@@ -77,7 +85,7 @@ public class DiffUtils {/*** Diff two list of strings.*/- public static List<DiffBlock<List<Token>>> diff(List<String> oldLines, @Nullable String oldFileName,+ public static List<DiffBlock<Tokenized>> diff(List<String> oldLines, @Nullable String oldFileName,List<String> newLines, @Nullable String newFileName, WhitespaceOption whitespaceOption) {Preconditions.checkArgument(oldLines.size() + newLines.size() <= MAX_DIFF_SIZE,"Total size of old lines and new lines should be less than " + MAX_DIFF_SIZE + ".");@@ -92,36 +100,36 @@ public class DiffUtils {processedNewLines.add(whitespaceOption.process(line));}- List<List<Token>> oldTokenizedLines = tokenize(oldLines, oldFileName);- List<List<Token>> newTokenizedLines = tokenize(newLines, newFileName);+ List<Tokenized> oldTokenizedLines = tokenize(oldLines, oldFileName);+ List<Tokenized> newTokenizedLines = tokenize(newLines, newFileName);DiffMatchPatch dmp = new DiffMatchPatch();TokensToCharsResult<String> result1 = tokensToChars(processedOldLines, processedNewLines);List<DiffMatchPatch.Diff> diffs = dmp.diff_main(result1.chars1, result1.chars2, false);- List<DiffBlock<List<Token>>> diffBlocks = new ArrayList<>();+ List<DiffBlock<Tokenized>> diffBlocks = new ArrayList<>();int oldLineNo = 0;int newLineNo = 0;for (Diff diff : diffs) {- List<List<Token>> lines = new ArrayList<>();+ List<Tokenized> lines = new ArrayList<>();if (diff.operation == Operation.EQUAL) {for (int i = 0; i < diff.text.length(); i++) {lines.add(newTokenizedLines.get(newLineNo));oldLineNo++;newLineNo++;}- diffBlocks.add(new DiffBlock<List<Token>>(diff.operation, lines,+ diffBlocks.add(new DiffBlock<>(diff.operation, lines,oldLineNo-lines.size(), newLineNo-lines.size()));} else if (diff.operation == Operation.INSERT) {for (int i = 0; i < diff.text.length(); i++)lines.add(newTokenizedLines.get(newLineNo++));- diffBlocks.add(new DiffBlock<List<Token>>(diff.operation, lines,+ diffBlocks.add(new DiffBlock<>(diff.operation, lines,oldLineNo, newLineNo-lines.size()));} else {for (int i = 0; i < diff.text.length(); i++)lines.add(oldTokenizedLines.get(oldLineNo++));- diffBlocks.add(new DiffBlock<List<Token>>(diff.operation, lines,+ diffBlocks.add(new DiffBlock<>(diff.operation, lines,oldLineNo-lines.size(), newLineNo));}}@@ -173,7 +181,7 @@ public class DiffUtils {* @return*/public static LinkedHashMap<Integer, LineDiff> align(- List<List<Token>> deleteLines, List<List<Token>> insertLines) {+ List<Tokenized> deleteLines, List<Tokenized> insertLines) {LinkedHashMap<Integer, LineDiff> lineDiffs = new LinkedHashMap<>();DiffMatchPatch dmp = new DiffMatchPatch();@@ -181,17 +189,20 @@ public class DiffUtils {long time = System.currentTimeMillis();int nextInsert = 0;for (int i=0; i<deleteLines.size(); i++) {- List<Token> deleteLine = deleteLines.get(i);+ Tokenized deleteLine = deleteLines.get(i);+ List<TextToken> deleteTokens = TokenUtils.getTextTokens(deleteLine);for (int j=nextInsert; j<insertLines.size(); j++) {- List<Token> insertLine = insertLines.get(j);- TokensToCharsResult<Token> result = DiffUtils.tokensToChars(deleteLine, insertLine);+ Tokenized insertLine = insertLines.get(j);+ List<TextToken> insertTokens = TokenUtils.getTextTokens(insertLine);++ TokensToCharsResult<TextToken> result = DiffUtils.tokensToChars(deleteTokens, insertTokens);List<DiffMatchPatch.Diff> diffs = dmp.diff_main(result.chars1, result.chars2, false);int equal = 0;int total = 0;for (DiffMatchPatch.Diff diff: diffs) {for (int k=0; k<diff.text.length(); k++) {int pos = diff.text.charAt(k);- Token token = result.tokenArray.get(pos);+ TextToken token = result.tokenArray.get(pos);if (StringUtils.isNotBlank(token.getText())) {total += token.getText().length();if (diff.operation == Operation.EQUAL)@@ -200,26 +211,29 @@ public class DiffUtils {}}if (equal*3 >= total) {- List<DiffBlock<Token>> diffBlocks = new ArrayList<>();+ List<DiffBlock<TextToken>> diffBlocks = new ArrayList<>();int oldLineNo = 0;int newLineNo = 0;for (Diff diff : diffs) {- List<Token> tokens = new ArrayList<>();+ List<TextToken> tokens = new ArrayList<>();if (diff.operation == Operation.EQUAL) {for (int k = 0; k < diff.text.length(); k++) {- tokens.add(insertLine.get(newLineNo));+ tokens.add(insertTokens.get(newLineNo));oldLineNo++;newLineNo++;}- diffBlocks.add(new DiffBlock<Token>(diff.operation, tokens, oldLineNo-tokens.size(), newLineNo-tokens.size()));+ diffBlocks.add(new DiffBlock<TextToken>(+ diff.operation, tokens, oldLineNo-tokens.size(), newLineNo-tokens.size()));} else if (diff.operation == Operation.INSERT) {for (int k = 0; k < diff.text.length(); k++)- tokens.add(insertLine.get(newLineNo++));- diffBlocks.add(new DiffBlock<Token>(diff.operation, tokens, oldLineNo, newLineNo-tokens.size()));+ tokens.add(insertTokens.get(newLineNo++));+ diffBlocks.add(new DiffBlock<>(+ diff.operation, tokens, oldLineNo, newLineNo-tokens.size()));} else {for (int k = 0; k < diff.text.length(); k++)- tokens.add(deleteLine.get(oldLineNo++));- diffBlocks.add(new DiffBlock<Token>(diff.operation, tokens, oldLineNo-tokens.size(), newLineNo));+ tokens.add(deleteTokens.get(oldLineNo++));+ diffBlocks.add(new DiffBlock<>(+ diff.operation, tokens, oldLineNo-tokens.size(), newLineNo));}}diff --git a/core/src/main/java/com/gitplex/server/util/diff/LineDiff.java b/core/src/main/java/com/gitplex/server/util/diff/LineDiff.javaindex a63238e69c..384a741905 100644--- a/core/src/main/java/com/gitplex/server/util/diff/LineDiff.java+++ b/core/src/main/java/com/gitplex/server/util/diff/LineDiff.java@@ -3,7 +3,7 @@ package com.gitplex.server.util.diff;import java.io.Serializable;import java.util.List;-import com.gitplex.jsyntax.Token;+import com.gitplex.jsyntax.TextToken;public class LineDiff implements Serializable {@@ -11,18 +11,18 @@ public class LineDiff implements Serializable {private final int compareLine;- private final List<DiffBlock<Token>> tokenDiffs;+ private final List<DiffBlock<TextToken>> tokenDiffs;- public LineDiff(int line, List<DiffBlock<Token>> diffs) {- this.compareLine = line;- this.tokenDiffs = diffs;+ public LineDiff(int compareLine, List<DiffBlock<TextToken>> tokenDiffs) {+ this.compareLine = compareLine;+ this.tokenDiffs = tokenDiffs;}public int getCompareLine() {return compareLine;}- public List<DiffBlock<Token>> getTokenDiffs() {+ public List<DiffBlock<TextToken>> getTokenDiffs() {return tokenDiffs;}diff --git a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/MarkAwareDiffBlock.java b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/MarkAwareDiffBlock.javaindex 6622311d80..82efbf738b 100644--- a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/MarkAwareDiffBlock.java+++ b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/MarkAwareDiffBlock.java@@ -3,7 +3,7 @@ package com.gitplex.server.web.component.diff.blob.text;import java.io.Serializable;import java.util.List;-import com.gitplex.jsyntax.Token;+import com.gitplex.jsyntax.Tokenized;public class MarkAwareDiffBlock implements Serializable {@@ -15,11 +15,11 @@ public class MarkAwareDiffBlock implements Serializable {private final Type type;- private final List<List<Token>> lines;+ private final List<Tokenized> lines;private final int oldStart, newStart;- public MarkAwareDiffBlock(Type type, List<List<Token>> lines, int oldStart, int newStart) {+ public MarkAwareDiffBlock(Type type, List<Tokenized> lines, int oldStart, int newStart) {this.type = type;this.lines = lines;this.oldStart = oldStart;@@ -30,7 +30,7 @@ public class MarkAwareDiffBlock implements Serializable {return type;}- public List<List<Token>> getLines() {+ public List<Tokenized> getLines() {return lines;}diff --git a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.javaindex 4c03464c49..7917f34132 100644--- a/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java+++ b/web/src/main/java/com/gitplex/server/web/component/diff/blob/text/TextDiffPanel.java@@ -37,7 +37,9 @@ import org.unbescape.html.HtmlEscape;import com.fasterxml.jackson.core.JsonProcessingException;import com.fasterxml.jackson.databind.ObjectMapper;import com.gitplex.jsymbol.Range;-import com.gitplex.jsyntax.Token;+import com.gitplex.jsyntax.TextToken;+import com.gitplex.jsyntax.TokenUtils;+import com.gitplex.jsyntax.Tokenized;import com.gitplex.server.GitPlex;import com.gitplex.server.git.Blame;import com.gitplex.server.git.BlobChange;@@ -624,7 +626,8 @@ public class TextDiffPanel extends Panel implements SourceAware {appendDeletesAndInserts(builder, block, nextBlock, prevDeleteLineIndex, deleteLineIndex,prevInsertLineIndex, insertLineIndex);- appendModification(builder, block, nextBlock, deleteLineIndex, insertLineIndex, lineChange.getTokenDiffs());+ appendModification(builder, block, nextBlock, deleteLineIndex, insertLineIndex,+ lineChange.getTokenDiffs());prevDeleteLineIndex = deleteLineIndex+1;prevInsertLineIndex = insertLineIndex+1;@@ -721,6 +724,16 @@ public class TextDiffPanel extends Panel implements SourceAware {return null;}+ private void appendTokenized(StringBuilder builder, Tokenized tokenized) {+ if (tokenized.getTokens().length == 0) {+ builder.append(" ");+ } else {+ for (long token: tokenized.getTokens()) {+ builder.append(TokenUtils.toHtml(tokenized.getText(), token, null));+ }+ }+ }+private void appendEqual(StringBuilder builder, MarkAwareDiffBlock block, int lineIndex, int lastContextSize) {if (lastContextSize != 0)builder.append("<tr class='code expanded'>");@@ -738,14 +751,7 @@ public class TextDiffPanel extends Panel implements SourceAware {builder.append("<td class='number noselect'>").append(newLineNo+1).append("</td>");builder.append("<td class='operation noselect'> </td>");builder.append("<td class='content' data-old='").append(oldLineNo).append("' data-new='").append(newLineNo).append("'>");- List<Token> tokens = block.getLines().get(lineIndex);- if (tokens.isEmpty()) {- builder.append(" ");- } else {- for (Token token: tokens) {- builder.append(token.toHtml(null));- }- }+ appendTokenized(builder, block.getLines().get(lineIndex));builder.append("</td>");} else {if (blameInfo != null) {@@ -754,14 +760,7 @@ public class TextDiffPanel extends Panel implements SourceAware {builder.append("<td class='number noselect'>").append(oldLineNo+1).append("</td>");builder.append("<td class='operation noselect'> </td>");builder.append("<td class='content left' data-old='").append(oldLineNo).append("' data-new='").append(newLineNo).append("'>");- List<Token> tokens = block.getLines().get(lineIndex);- if (tokens.isEmpty()) {- builder.append(" ");- } else {- for (Token token: tokens) {- builder.append(token.toHtml(null));- }- }+ appendTokenized(builder, block.getLines().get(lineIndex));builder.append("</td>");if (blameInfo != null) {appendBlame(builder, -1, newLineNo);@@ -769,20 +768,14 @@ public class TextDiffPanel extends Panel implements SourceAware {builder.append("<td class='number noselect'>").append(newLineNo+1).append("</td>");builder.append("<td class='operation noselect'> </td>");builder.append("<td class='content right' data-old='").append(oldLineNo).append("' data-new='").append(newLineNo).append("'>");- tokens = block.getLines().get(lineIndex);- if (tokens.isEmpty()) {- builder.append(" ");- } else {- for (Token token: tokens) {- builder.append(token.toHtml(null));- }- }+ appendTokenized(builder, block.getLines().get(lineIndex));builder.append("</td>");}builder.append("</tr>");}- private void appendInsert(StringBuilder builder, MarkAwareDiffBlock block, int lineIndex, @Nullable List<DiffBlock<Token>> tokenDiffs) {+ private void appendInsert(StringBuilder builder, MarkAwareDiffBlock block, int lineIndex,+ @Nullable List<DiffBlock<TextToken>> tokenDiffs) {builder.append("<tr class='code original'>");int newLineNo = block.getNewStart() + lineIndex;@@ -798,21 +791,15 @@ public class TextDiffPanel extends Panel implements SourceAware {if (tokenDiffs.isEmpty()) {builder.append(" ");} else {- for (DiffBlock<Token> tokenBlock: tokenDiffs) {- for (Token token: tokenBlock.getUnits()) {+ for (DiffBlock<TextToken> tokenBlock: tokenDiffs) {+ for (TextToken token: tokenBlock.getUnits()) {if (tokenBlock.getOperation() != Operation.DELETE)- builder.append(token.toHtml(getOperationClass(tokenBlock.getOperation())));+ builder.append(TokenUtils.toHtml(token, getOperationClass(tokenBlock.getOperation())));}}}} else {- List<Token> tokens = block.getLines().get(lineIndex);- if (tokens.isEmpty()) {- builder.append(" ");- } else {- for (int i=0; i<tokens.size(); i++)- builder.append(tokens.get(i).toHtml(null));- }+ appendTokenized(builder, block.getLines().get(lineIndex));}builder.append("</td>");} else {@@ -828,19 +815,14 @@ public class TextDiffPanel extends Panel implements SourceAware {builder.append("<td class='number noselect new'>").append(newLineNo+1).append("</td>");builder.append("<td class='operation noselect new'>+</td>");builder.append("<td class='content right new' data-new='").append(newLineNo).append("'>");- List<Token> tokens = block.getLines().get(lineIndex);- if (tokens.isEmpty()) {- builder.append(" ");- } else {- for (int i=0; i<tokens.size(); i++)- builder.append(tokens.get(i).toHtml(null));- }+ appendTokenized(builder, block.getLines().get(lineIndex));builder.append("</td>");}builder.append("</tr>");}- private void appendDelete(StringBuilder builder, MarkAwareDiffBlock block, int lineIndex, @Nullable List<DiffBlock<Token>> tokenDiffs) {+ private void appendDelete(StringBuilder builder, MarkAwareDiffBlock block, int lineIndex,+ @Nullable List<DiffBlock<TextToken>> tokenDiffs) {builder.append("<tr class='code original'>");int oldLineNo = block.getOldStart() + lineIndex;@@ -856,21 +838,15 @@ public class TextDiffPanel extends Panel implements SourceAware {if (tokenDiffs.isEmpty()) {builder.append(" ");} else {- for (DiffBlock<Token> tokenBlock: tokenDiffs) {- for (Token token: tokenBlock.getUnits()) {+ for (DiffBlock<TextToken> tokenBlock: tokenDiffs) {+ for (TextToken token: tokenBlock.getUnits()) {if (tokenBlock.getOperation() != Operation.INSERT)- builder.append(token.toHtml(getOperationClass(tokenBlock.getOperation())));+ builder.append(TokenUtils.toHtml(token, getOperationClass(tokenBlock.getOperation())));}}}} else {- List<Token> tokens = block.getLines().get(lineIndex);- if (tokens.isEmpty()) {- builder.append(" ");- } else {- for (int i=0; i<tokens.size(); i++)- builder.append(tokens.get(i).toHtml(null));- }+ appendTokenized(builder, block.getLines().get(lineIndex));}builder.append("</td>");} else {@@ -880,13 +856,7 @@ public class TextDiffPanel extends Panel implements SourceAware {builder.append("<td class='number noselect old'>").append(oldLineNo+1).append("</td>");builder.append("<td class='operation noselect old'>-</td>");builder.append("<td class='content left old' data-old='").append(oldLineNo).append("'>");- List<Token> tokens = block.getLines().get(lineIndex);- if (tokens.isEmpty()) {- builder.append(" ");- } else {- for (int i=0; i<tokens.size(); i++)- builder.append(tokens.get(i).toHtml(null));- }+ appendTokenized(builder, block.getLines().get(lineIndex));builder.append("</td>");if (blameInfo != null) {builder.append("<td class='blame noselect'> </td>");@@ -909,13 +879,7 @@ public class TextDiffPanel extends Panel implements SourceAware {builder.append("<td class='number noselect old'>").append(oldLineNo+1).append("</td>");builder.append("<td class='operation noselect old'>-</td>");builder.append("<td class='content left old' data-old='").append(oldLineNo).append("'>");- List<Token> tokens = deleteBlock.getLines().get(deleteLineIndex);- if (tokens.isEmpty()) {- builder.append(" ");- } else {- for (Token token: tokens)- builder.append(token.toHtml(null));- }+ appendTokenized(builder, deleteBlock.getLines().get(deleteLineIndex));builder.append("</td>");int newLineNo = insertBlock.getNewStart()+insertLineIndex;@@ -925,13 +889,7 @@ public class TextDiffPanel extends Panel implements SourceAware {builder.append("<td class='number noselect new'>").append(newLineNo+1).append("</td>");builder.append("<td class='operation noselect new'>+</td>");builder.append("<td class='content right new' data-new='").append(newLineNo).append("'>");- tokens = insertBlock.getLines().get(insertLineIndex);- if (tokens.isEmpty()) {- builder.append(" ");- } else {- for (Token token: tokens)- builder.append(token.toHtml(null));- }+ appendTokenized(builder, insertBlock.getLines().get(insertLineIndex));builder.append("</td>");builder.append("</tr>");@@ -939,7 +897,7 @@ public class TextDiffPanel extends Panel implements SourceAware {private void appendModification(StringBuilder builder, MarkAwareDiffBlock deleteBlock,MarkAwareDiffBlock insertBlock, int deleteLineIndex, int insertLineIndex,- List<DiffBlock<Token>> tokenDiffs) {+ List<DiffBlock<TextToken>> tokenDiffs) {builder.append("<tr class='code original'>");int oldLineNo = deleteBlock.getOldStart() + deleteLineIndex;@@ -955,9 +913,9 @@ public class TextDiffPanel extends Panel implements SourceAware {if (tokenDiffs.isEmpty()) {builder.append(" ");} else {- for (DiffBlock<Token> tokenBlock: tokenDiffs) {- for (Token token: tokenBlock.getUnits())- builder.append(token.toHtml(getOperationClass(tokenBlock.getOperation())));+ for (DiffBlock<TextToken> tokenBlock: tokenDiffs) {+ for (TextToken token: tokenBlock.getUnits())+ builder.append(TokenUtils.toHtml(token, getOperationClass(tokenBlock.getOperation())));}}builder.append("</td>");@@ -971,10 +929,10 @@ public class TextDiffPanel extends Panel implements SourceAware {if (tokenDiffs.isEmpty()) {builder.append(" ");} else {- for (DiffBlock<Token> tokenBlock: tokenDiffs) {- for (Token token: tokenBlock.getUnits()) {+ for (DiffBlock<TextToken> tokenBlock: tokenDiffs) {+ for (TextToken token: tokenBlock.getUnits()) {if (tokenBlock.getOperation() != Operation.INSERT)- builder.append(token.toHtml(getOperationClass(tokenBlock.getOperation())));+ builder.append(TokenUtils.toHtml(token, getOperationClass(tokenBlock.getOperation())));}}}@@ -989,10 +947,10 @@ public class TextDiffPanel extends Panel implements SourceAware {if (tokenDiffs.isEmpty()) {builder.append(" ");} else {- for (DiffBlock<Token> tokenBlock: tokenDiffs) {- for (Token token: tokenBlock.getUnits()) {+ for (DiffBlock<TextToken> tokenBlock: tokenDiffs) {+ for (TextToken token: tokenBlock.getUnits()) {if (tokenBlock.getOperation() != Operation.DELETE)- builder.append(token.toHtml(getOperationClass(tokenBlock.getOperation())));+ builder.append(TokenUtils.toHtml(token, getOperationClass(tokenBlock.getOperation())));}}}@@ -1064,7 +1022,7 @@ public class TextDiffPanel extends Panel implements SourceAware {}}- for (DiffBlock<List<Token>> diffBlock: change.getDiffBlocks()) {+ for (DiffBlock<Tokenized> diffBlock: change.getDiffBlocks()) {if (diffBlock.getOperation() == Operation.DELETE) {diffBlocks.add(new MarkAwareDiffBlock(Type.DELETE, diffBlock.getUnits(),diffBlock.getOldStart(), diffBlock.getNewStart()));@@ -1091,12 +1049,12 @@ public class TextDiffPanel extends Panel implements SourceAware {if (to > diffBlock.getUnits().size())to = diffBlock.getUnits().size();if (lastIndex < from) {- List<List<Token>> lines = diffBlock.getUnits().subList(lastIndex, from);+ List<Tokenized> lines = diffBlock.getUnits().subList(lastIndex, from);diffBlocks.add(new MarkAwareDiffBlock(Type.EQUAL, lines,diffBlock.getOldStart()+lastIndex, diffBlock.getNewStart()+lastIndex));}if (from < to) {- List<List<Token>> lines = diffBlock.getUnits().subList(from, to);+ List<Tokenized> lines = diffBlock.getUnits().subList(from, to);diffBlocks.add(new MarkAwareDiffBlock(Type.MARKED_EQUAL, lines,diffBlock.getOldStart()+from, diffBlock.getNewStart()+from));}@@ -1105,7 +1063,7 @@ public class TextDiffPanel extends Panel implements SourceAware {}if (lastIndex < diffBlock.getUnits().size()) {- List<List<Token>> lines = diffBlock.getUnits().subList(lastIndex, diffBlock.getUnits().size());+ List<Tokenized> lines = diffBlock.getUnits().subList(lastIndex, diffBlock.getUnits().size());diffBlocks.add(new MarkAwareDiffBlock(Type.EQUAL, lines,diffBlock.getOldStart()+lastIndex, diffBlock.getNewStart()+lastIndex));}