Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9531: Consolidate duplicated generated classes CharStream and FastCharStream (#1886)

Dawid Weiss5ec2bac91c5e
diff --git a/gradle/generation/javacc.gradle b/gradle/generation/javacc.gradle
index 90eed8de0cf..d2b50937195 100644
--- a/gradle/generation/javacc.gradle
+++ b/gradle/generation/javacc.gradle
@@ -73,6 +73,17 @@ def commonCleanups = { FileTree generatedFiles ->
generatedFiles.matching({ include "*TokenManager.java" }).each { file ->
modifyFile(file, { text ->
+ // Remove redundant imports.
+ text = text.replaceAll(
+ /(?m)^import .+/,
+ "")
+ // Add CharStream imports.
+ text = text.replaceAll(
+ /package (.+)/,
+ '''
+ package $1
+ import org.apache.lucene.queryparser.charstream.CharStream;
+ '''.trim())
// Eliminates redundant cast message.
text = text.replace(
"int hiByte = (int)(curChar >> 8);",
@@ -112,16 +123,6 @@ configure(project(":lucene:queryparser")) {
return text
})
}
-
- generatedFiles.matching { include "*TokenManager.java" }.each { file ->
- modifyFile(file, { text ->
- // Remove redundant imports.
- text = text.replaceAll(
- /(?m)^import .+/,
- "")
- return text
- })
- }
}
}
@@ -144,16 +145,6 @@ configure(project(":lucene:queryparser")) {
return text
})
}
-
- generatedFiles.matching { include "*TokenManager.java" }.each { file ->
- modifyFile(file, { text ->
- // Remove redundant imports.
- text = text.replaceAll(
- /(?m)^import .+/,
- "")
- return text
- })
- }
}
}
@@ -230,16 +221,6 @@ configure(project(":lucene:queryparser")) {
return text
})
}
-
- generatedFiles.matching { include "StandardSyntaxParserTokenManager.java" }.each { file ->
- modifyFile(file, { text ->
- // Remove redundant imports.
- text = text.replaceAll(
- /(?m)^import .+/,
- "")
- return text
- })
- }
}
}
@@ -277,16 +258,6 @@ configure(project(":solr:core")) {
return text
})
}
-
- generatedFiles.matching { include "*TokenManager.java" }.each { file ->
- modifyFile(file, { text ->
- // Remove redundant imports.
- text = text.replaceAll(
- /(?m)^import .+/,
- "")
- return text
- })
- }
}
}
}
@@ -363,6 +334,9 @@ class JavaCCTask extends DefaultTask {
project.copy {
from tempDir
into targetDir
+
+ // We don't need CharStream interface as we redirect to our own.
+ exclude "CharStream.java"
}
}
}
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b60ab9800b3..a4278ab517d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -134,6 +134,9 @@ Improvements
* LUCENE-9527: Upgrade javacc to 7.0.4, regenerate query parsers. (Dawid Weiss)
+* LUCENE-9531: Consolidated CharStream and FastCharStream classes: these have been moved
+ from each query parser package to org.apache.lucene.queryparser.charstream (Dawid Weiss).
+
Bug fixes
* LUCENE-8663: NRTCachingDirectory.slowFileExists may open a file while
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/CharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/CharStream.java
similarity index 78%
rename from lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/CharStream.java
rename to lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/CharStream.java
index 81567ff98cf..1a293afd985 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/CharStream.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/CharStream.java
@@ -1,6 +1,20 @@
-/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 7.0 */
-/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
-package org.apache.lucene.queryparser.classic;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.queryparser.charstream;
/**
* This interface describes a character stream that maintains line and
@@ -16,9 +30,7 @@ package org.apache.lucene.queryparser.classic;
* by the lexer. Hence their implementation won't affect the generated lexer's
* operation.
*/
-
-public
-interface CharStream {
+public interface CharStream {
/**
* Returns the next character from the selected input. The method
@@ -27,22 +39,6 @@ interface CharStream {
*/
char readChar() throws java.io.IOException;
- @Deprecated
- /**
- * Returns the column position of the character last read.
- * @deprecated
- * @see #getEndColumn
- */
- int getColumn();
-
- @Deprecated
- /**
- * Returns the line number of the character last read.
- * @deprecated
- * @see #getEndLine
- */
- int getLine();
-
/**
* Returns the column number of the last character for current token (being
* matched after the last call to BeginTOken).
@@ -110,11 +106,4 @@ interface CharStream {
* affect the lexer's operation.
*/
void Done();
-
-
- void setTabSize(int i);
- int getTabSize();
- boolean getTrackLineColumn();
- void setTrackLineColumn(boolean trackLineColumn);
}
-/* (filtered)*/
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/FastCharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/FastCharStream.java
similarity index 72%
rename from lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/FastCharStream.java
rename to lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/FastCharStream.java
index c49548b04a2..f48996fed29 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/FastCharStream.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/FastCharStream.java
@@ -14,15 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.queryparser.surround.parser;
+package org.apache.lucene.queryparser.charstream;
import java.io.*;
-/** An efficient implementation of JavaCC's CharStream interface. <p>Note that
- * this does not do line-number counting, but instead keeps track of the
- * character position of the token in the input, as required by Lucene's {@link
- * org.apache.lucene.analysis.tokenattributes.OffsetAttribute} API. */
+/**
+ * An efficient implementation of JavaCC's CharStream interface.
+ * <p>
+ * Note that this does not do line-number counting, but instead keeps track of the
+ * character position of the token in the input, as required by Lucene's
+ * {@link org.apache.lucene.analysis.tokenattributes.OffsetAttribute} API.
+ */
public final class FastCharStream implements CharStream {
+ // See SOLR-11314
+ private final static IOException READ_PAST_EOF = new IOException("Read past EOF.");
+
char[] buffer = null;
int bufferLength = 0; // end of valid chars
@@ -33,7 +39,9 @@ public final class FastCharStream implements CharStream {
Reader input; // source of chars
- /** Constructs from a Reader. */
+ /**
+ * Constructs from a Reader.
+ */
public FastCharStream(Reader r) {
input = r;
}
@@ -45,7 +53,7 @@ public final class FastCharStream implements CharStream {
return buffer[bufferPosition++];
}
- private final void refill() throws IOException {
+ private void refill() throws IOException {
int newPosition = bufferLength - tokenStart;
if (tokenStart == 0) { // token won't fit in buffer
@@ -66,9 +74,9 @@ public final class FastCharStream implements CharStream {
tokenStart = 0;
int charsRead = // fill space in buffer
- input.read(buffer, newPosition, buffer.length-newPosition);
+ input.read(buffer, newPosition, buffer.length - newPosition);
if (charsRead == -1)
- throw new IOException("read past eof");
+ throw READ_PAST_EOF;
else
bufferLength += charsRead;
}
@@ -101,60 +109,27 @@ public final class FastCharStream implements CharStream {
try {
input.close();
} catch (IOException e) {
+ throw new UncheckedIOException(e);
}
}
- /**
- * Returns the column position of the character last read.
- * @deprecated see: #getEndColumn
- */
- @Deprecated
- @Override
- public final int getColumn() {
- return bufferStart + bufferPosition;
- }
- /**
- * @deprecated see: #getEndLine
- */
- @Deprecated
- @Override
- public final int getLine() {
- return 1;
- }
@Override
public final int getEndColumn() {
return bufferStart + bufferPosition;
}
+
@Override
public final int getEndLine() {
return 1;
}
+
@Override
public final int getBeginColumn() {
return bufferStart + tokenStart;
}
+
@Override
public final int getBeginLine() {
return 1;
}
-
- @Override
- public void setTabSize(int i) {
- throw new RuntimeException("Tab size not implemented.");
- }
-
- @Override
- public int getTabSize() {
- throw new RuntimeException("Tab size not implemented.");
- }
-
- @Override
- public boolean getTrackLineColumn() {
- return false;
- }
-
- @Override
- public void setTrackLineColumn(boolean trackLineColumn) {
- throw new RuntimeException("Line/Column tracking not implemented.");
- }
}
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/package-info.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/package-info.java
new file mode 100644
index 00000000000..dc30e767015
--- /dev/null
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/charstream/package-info.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * This package contains reusable parts for javacc-generated
+ * grammars (query parsers).
+ *
+ * @see org.apache.lucene.queryparser.charstream.CharStream
+ * @see org.apache.lucene.queryparser.charstream.FastCharStream
+ */
+package org.apache.lucene.queryparser.charstream;
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/FastCharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/FastCharStream.java
deleted file mode 100644
index e01b90a60ea..00000000000
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/FastCharStream.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.queryparser.classic;
-
-import java.io.*;
-
-/** An efficient implementation of JavaCC's CharStream interface. <p>Note that
- * this does not do line-number counting, but instead keeps track of the
- * character position of the token in the input, as required by Lucene's {@link
- * org.apache.lucene.analysis.tokenattributes.OffsetAttribute} API.
- * */
-public final class FastCharStream implements CharStream {
- char[] buffer = null;
-
- int bufferLength = 0; // end of valid chars
- int bufferPosition = 0; // next char to read
-
- int tokenStart = 0; // offset in buffer
- int bufferStart = 0; // position in file of buffer
-
- Reader input; // source of chars
-
- /** Constructs from a Reader. */
- public FastCharStream(Reader r) {
- input = r;
- }
-
- @Override
- public final char readChar() throws IOException {
- if (bufferPosition >= bufferLength)
- refill();
- return buffer[bufferPosition++];
- }
-
- private final void refill() throws IOException {
- int newPosition = bufferLength - tokenStart;
-
- if (tokenStart == 0) { // token won't fit in buffer
- if (buffer == null) { // first time: alloc buffer
- buffer = new char[2048];
- } else if (bufferLength == buffer.length) { // grow buffer
- char[] newBuffer = new char[buffer.length*2];
- System.arraycopy(buffer, 0, newBuffer, 0, bufferLength);
- buffer = newBuffer;
- }
- } else { // shift token to front
- System.arraycopy(buffer, tokenStart, buffer, 0, newPosition);
- }
-
- bufferLength = newPosition; // update state
- bufferPosition = newPosition;
- bufferStart += tokenStart;
- tokenStart = 0;
-
- int charsRead = // fill space in buffer
- input.read(buffer, newPosition, buffer.length-newPosition);
- if (charsRead == -1)
- throw new IOException("read past eof");
- else
- bufferLength += charsRead;
- }
-
- @Override
- public final char BeginToken() throws IOException {
- tokenStart = bufferPosition;
- return readChar();
- }
-
- @Override
- public final void backup(int amount) {
- bufferPosition -= amount;
- }
-
- @Override
- public final String GetImage() {
- return new String(buffer, tokenStart, bufferPosition - tokenStart);
- }
-
- @Override
- public final char[] GetSuffix(int len) {
- char[] value = new char[len];
- System.arraycopy(buffer, bufferPosition - len, value, 0, len);
- return value;
- }
-
- @Override
- public final void Done() {
- try {
- input.close();
- } catch (IOException e) {
- }
- }
-
- /**
- * Returns the column position of the character last read.
- * @deprecated see: #getEndColumn
- */
- @Deprecated
- @Override
- public final int getColumn() {
- return bufferStart + bufferPosition;
- }
- /**
- * @deprecated see #getEndLine
- */
- @Override
- @Deprecated
- public final int getLine() {
- return 1;
- }
- @Override
- public final int getEndColumn() {
- return bufferStart + bufferPosition;
- }
- @Override
- public final int getEndLine() {
- return 1;
- }
- @Override
- public final int getBeginColumn() {
- return bufferStart + tokenStart;
- }
- @Override
- public final int getBeginLine() {
- return 1;
- }
-
- @Override
- public void setTabSize(int i) {
- throw new RuntimeException("Tab size not implemented.");
- }
-
- @Override
- public int getTabSize() {
- throw new RuntimeException("Tab size not implemented.");
- }
-
- @Override
- public boolean getTrackLineColumn() {
- return false;
- }
-
- @Override
- public void setTrackLineColumn(boolean trackLineColumn) {
- throw new RuntimeException("Line/Column tracking not implemented.");
- }
-}
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java
index f7da7e01889..27488161260 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.java
@@ -15,6 +15,8 @@ import org.apache.lucene.document.DateTools;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.queryparser.charstream.CharStream;
+import org.apache.lucene.queryparser.charstream.FastCharStream;
/**
* This class is generated by JavaCC. The most important method is
@@ -706,22 +708,6 @@ if (splitOnWhitespace == false) {
finally { jj_save(2, xla); }
}
- private boolean jj_3R_3()
- {
- if (jj_scan_token(TERM)) return true;
- jj_lookingAhead = true;
- jj_semLA = getToken(1).kind == TERM && allowedPostMultiTerm(getToken(2).kind);
- jj_lookingAhead = false;
- if (!jj_semLA || jj_3R_6()) return true;
- Token xsp;
- if (jj_3R_7()) return true;
- while (true) {
- xsp = jj_scanpos;
- if (jj_3R_7()) { jj_scanpos = xsp; break; }
- }
- return false;
- }
-
private boolean jj_3R_6()
{
return false;
@@ -770,6 +756,22 @@ if (splitOnWhitespace == false) {
return false;
}
+ private boolean jj_3R_3()
+ {
+ if (jj_scan_token(TERM)) return true;
+ jj_lookingAhead = true;
+ jj_semLA = getToken(1).kind == TERM && allowedPostMultiTerm(getToken(2).kind);
+ jj_lookingAhead = false;
+ if (!jj_semLA || jj_3R_6()) return true;
+ Token xsp;
+ if (jj_3R_7()) return true;
+ while (true) {
+ xsp = jj_scanpos;
+ if (jj_3R_7()) { jj_scanpos = xsp; break; }
+ }
+ return false;
+ }
+
/** Generated Token Manager. */
public QueryParserTokenManager token_source;
/** Current token. */
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj
index fdc0cd0ed5e..11ba876b332 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj
@@ -38,6 +38,8 @@ import org.apache.lucene.document.DateTools;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.queryparser.charstream.CharStream;
+import org.apache.lucene.queryparser.charstream.FastCharStream;
/**
* This class is generated by JavaCC. The most important method is
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
index a1f8fa582ad..6afe913d0b1 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
@@ -25,6 +25,8 @@ import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.charstream.CharStream;
+import org.apache.lucene.queryparser.charstream.FastCharStream;
import org.apache.lucene.queryparser.classic.QueryParser.Operator;
import org.apache.lucene.queryparser.flexible.standard.CommonQueryParserConfiguration;
import org.apache.lucene.search.*;
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java
index db84cc7d006..10d09505d25 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java
@@ -1,6 +1,9 @@
/* QueryParserTokenManager.java */
/* Generated By:JavaCC: Do not edit this line. QueryParserTokenManager.java */
package org.apache.lucene.queryparser.classic;
+ import org.apache.lucene.queryparser.charstream.CharStream;
+
+
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java
deleted file mode 100644
index 575be8e49aa..00000000000
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 7.0 */
-/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
-package org.apache.lucene.queryparser.flexible.standard.parser;
-
-/**
- * This interface describes a character stream that maintains line and
- * column number positions of the characters. It also has the capability
- * to backup the stream to some extent. An implementation of this
- * interface is used in the TokenManager implementation generated by
- * JavaCCParser.
- *
- * All the methods except backup can be implemented in any fashion. backup
- * needs to be implemented correctly for the correct operation of the lexer.
- * Rest of the methods are all used to get information like line number,
- * column number and the String that constitutes a token and are not used
- * by the lexer. Hence their implementation won't affect the generated lexer's
- * operation.
- */
-
-public
-interface CharStream {
-
- /**
- * Returns the next character from the selected input. The method
- * of selecting the input is the responsibility of the class
- * implementing this interface. Can throw any java.io.IOException.
- */
- char readChar() throws java.io.IOException;
-
- @Deprecated
- /**
- * Returns the column position of the character last read.
- * @deprecated
- * @see #getEndColumn
- */
- int getColumn();
-
- @Deprecated
- /**
- * Returns the line number of the character last read.
- * @deprecated
- * @see #getEndLine
- */
- int getLine();
-
- /**
- * Returns the column number of the last character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getEndColumn();
-
- /**
- * Returns the line number of the last character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getEndLine();
-
- /**
- * Returns the column number of the first character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getBeginColumn();
-
- /**
- * Returns the line number of the first character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getBeginLine();
-
- /**
- * Backs up the input stream by amount steps. Lexer calls this method if it
- * had already read some characters, but could not use them to match a
- * (longer) token. So, they will be used again as the prefix of the next
- * token and it is the implementation's responsibility to do this right.
- */
- void backup(int amount);
-
- /**
- * Returns the next character that marks the beginning of the next token.
- * All characters must remain in the buffer between two successive calls
- * to this method to implement backup correctly.
- */
- char BeginToken() throws java.io.IOException;
-
- /**
- * Returns a string made up of characters from the marked token beginning
- * to the current buffer position. Implementations have the choice of returning
- * anything that they want to. For example, for efficiency, one might decide
- * to just return null, which is a valid implementation.
- */
- String GetImage();
-
- /**
- * Returns an array of characters that make up the suffix of length 'len' for
- * the currently matched token. This is used to build up the matched string
- * for use in actions in the case of MORE. A simple and inefficient
- * implementation of this is as follows :
- *
- * {
- * String t = GetImage();
- * return t.substring(t.length() - len, t.length()).toCharArray();
- * }
- */
- char[] GetSuffix(int len);
-
- /**
- * The lexer calls this function to indicate that it is done with the stream
- * and hence implementations can free any resources held by this class.
- * Again, the body of this function can be just empty and it will not
- * affect the lexer's operation.
- */
- void Done();
-
-
- void setTabSize(int i);
- int getTabSize();
- boolean getTrackLineColumn();
- void setTrackLineColumn(boolean trackLineColumn);
-}
-/* (filtered)*/
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/FastCharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/FastCharStream.java
deleted file mode 100644
index 7b206fc1155..00000000000
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/FastCharStream.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.queryparser.flexible.standard.parser;
-
-import java.io.*;
-
-/** An efficient implementation of JavaCC's CharStream interface. <p>Note that
- * this does not do line-number counting, but instead keeps track of the
- * character position of the token in the input, as required by Lucene's {@link
- * org.apache.lucene.analysis.tokenattributes.OffsetAttribute} API.
- * */
-public final class FastCharStream implements CharStream {
- char[] buffer = null;
-
- int bufferLength = 0; // end of valid chars
- int bufferPosition = 0; // next char to read
-
- int tokenStart = 0; // offset in buffer
- int bufferStart = 0; // position in file of buffer
-
- Reader input; // source of chars
-
- /** Constructs from a Reader. */
- public FastCharStream(Reader r) {
- input = r;
- }
-
- @Override
- public final char readChar() throws IOException {
- if (bufferPosition >= bufferLength)
- refill();
- return buffer[bufferPosition++];
- }
-
- private final void refill() throws IOException {
- int newPosition = bufferLength - tokenStart;
-
- if (tokenStart == 0) { // token won't fit in buffer
- if (buffer == null) { // first time: alloc buffer
- buffer = new char[2048];
- } else if (bufferLength == buffer.length) { // grow buffer
- char[] newBuffer = new char[buffer.length * 2];
- System.arraycopy(buffer, 0, newBuffer, 0, bufferLength);
- buffer = newBuffer;
- }
- } else { // shift token to front
- System.arraycopy(buffer, tokenStart, buffer, 0, newPosition);
- }
-
- bufferLength = newPosition; // update state
- bufferPosition = newPosition;
- bufferStart += tokenStart;
- tokenStart = 0;
-
- int charsRead = // fill space in buffer
- input.read(buffer, newPosition, buffer.length-newPosition);
- if (charsRead == -1)
- throw new IOException("read past eof");
- else
- bufferLength += charsRead;
- }
-
- @Override
- public final char BeginToken() throws IOException {
- tokenStart = bufferPosition;
- return readChar();
- }
-
- @Override
- public final void backup(int amount) {
- bufferPosition -= amount;
- }
-
- @Override
- public final String GetImage() {
- return new String(buffer, tokenStart, bufferPosition - tokenStart);
- }
-
- @Override
- public final char[] GetSuffix(int len) {
- char[] value = new char[len];
- System.arraycopy(buffer, bufferPosition - len, value, 0, len);
- return value;
- }
-
- @Override
- public final void Done() {
- try {
- input.close();
- } catch (IOException e) {
- }
- }
-
- /**
- * Returns the column position of the character last read.
- * @deprecated see #getEndColumn
- */
- @Deprecated
- @Override
- public final int getColumn() {
- return bufferStart + bufferPosition;
- }
- /**
- * @deprecated see #getEndLine
- */
- @Deprecated
- @Override
- public final int getLine() {
- return 1;
- }
- @Override
- public final int getEndColumn() {
- return bufferStart + bufferPosition;
- }
- @Override
- public final int getEndLine() {
- return 1;
- }
- @Override
- public final int getBeginColumn() {
- return bufferStart + tokenStart;
- }
- @Override
- public final int getBeginLine() {
- return 1;
- }
-
- @Override
- public void setTabSize(int i) {
- throw new RuntimeException("Tab size not implemented.");
- }
-
- @Override
- public int getTabSize() {
- throw new RuntimeException("Tab size not implemented.");
- }
-
- @Override
- public boolean getTrackLineColumn() {
- return false;
- }
-
- @Override
- public void setTrackLineColumn(boolean trackLineColumn) {
- throw new RuntimeException("Line/Column tracking not implemented.");
- }
-}
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java
index 66500fac40d..4a5667f2001 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.java
@@ -41,6 +41,8 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.QuotedFieldQueryNode;
import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser;
import org.apache.lucene.queryparser.flexible.standard.nodes.TermRangeQueryNode;
+import org.apache.lucene.queryparser.charstream.CharStream;
+import org.apache.lucene.queryparser.charstream.FastCharStream;
/**
* Parser for the standard Lucene syntax
@@ -747,17 +749,6 @@ if (boost != null) {
finally { jj_save(2, xla); }
}
- private boolean jj_3R_26()
- {
- Token xsp;
- xsp = jj_scanpos;
- if (jj_3_2()) {
- jj_scanpos = xsp;
- if (jj_3_3()) return true;
- }
- return false;
- }
-
private boolean jj_3R_22()
{
if (jj_scan_token(OR)) return true;
@@ -1045,6 +1036,17 @@ if (boost != null) {
return false;
}
+ private boolean jj_3R_26()
+ {
+ Token xsp;
+ xsp = jj_scanpos;
+ if (jj_3_2()) {
+ jj_scanpos = xsp;
+ if (jj_3_3()) return true;
+ }
+ return false;
+ }
+
/** Generated Token Manager. */
public StandardSyntaxParserTokenManager token_source;
/** Current token. */
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj
index 69d10847a79..a25de283b51 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj
@@ -48,6 +48,8 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.QuotedFieldQueryNode;
import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser;
import org.apache.lucene.queryparser.flexible.standard.nodes.TermRangeQueryNode;
+import org.apache.lucene.queryparser.charstream.CharStream;
+import org.apache.lucene.queryparser.charstream.FastCharStream;
/**
* Parser for the standard Lucene syntax
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java
index bbfa11f0e01..2860fae683c 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParserTokenManager.java
@@ -1,6 +1,7 @@
/* StandardSyntaxParserTokenManager.java */
/* Generated By:JavaCC: Do not edit this line. StandardSyntaxParserTokenManager.java */
package org.apache.lucene.queryparser.flexible.standard.parser;
+ import org.apache.lucene.queryparser.charstream.CharStream;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -37,6 +38,8 @@ package org.apache.lucene.queryparser.flexible.standard.parser;
+
+
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java
deleted file mode 100644
index 4481de6c663..00000000000
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 7.0 */
-/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
-package org.apache.lucene.queryparser.surround.parser;
-
-/**
- * This interface describes a character stream that maintains line and
- * column number positions of the characters. It also has the capability
- * to backup the stream to some extent. An implementation of this
- * interface is used in the TokenManager implementation generated by
- * JavaCCParser.
- *
- * All the methods except backup can be implemented in any fashion. backup
- * needs to be implemented correctly for the correct operation of the lexer.
- * Rest of the methods are all used to get information like line number,
- * column number and the String that constitutes a token and are not used
- * by the lexer. Hence their implementation won't affect the generated lexer's
- * operation.
- */
-
-public
-interface CharStream {
-
- /**
- * Returns the next character from the selected input. The method
- * of selecting the input is the responsibility of the class
- * implementing this interface. Can throw any java.io.IOException.
- */
- char readChar() throws java.io.IOException;
-
- @Deprecated
- /**
- * Returns the column position of the character last read.
- * @deprecated
- * @see #getEndColumn
- */
- int getColumn();
-
- @Deprecated
- /**
- * Returns the line number of the character last read.
- * @deprecated
- * @see #getEndLine
- */
- int getLine();
-
- /**
- * Returns the column number of the last character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getEndColumn();
-
- /**
- * Returns the line number of the last character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getEndLine();
-
- /**
- * Returns the column number of the first character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getBeginColumn();
-
- /**
- * Returns the line number of the first character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getBeginLine();
-
- /**
- * Backs up the input stream by amount steps. Lexer calls this method if it
- * had already read some characters, but could not use them to match a
- * (longer) token. So, they will be used again as the prefix of the next
- * token and it is the implementation's responsibility to do this right.
- */
- void backup(int amount);
-
- /**
- * Returns the next character that marks the beginning of the next token.
- * All characters must remain in the buffer between two successive calls
- * to this method to implement backup correctly.
- */
- char BeginToken() throws java.io.IOException;
-
- /**
- * Returns a string made up of characters from the marked token beginning
- * to the current buffer position. Implementations have the choice of returning
- * anything that they want to. For example, for efficiency, one might decide
- * to just return null, which is a valid implementation.
- */
- String GetImage();
-
- /**
- * Returns an array of characters that make up the suffix of length 'len' for
- * the currently matched token. This is used to build up the matched string
- * for use in actions in the case of MORE. A simple and inefficient
- * implementation of this is as follows :
- *
- * {
- * String t = GetImage();
- * return t.substring(t.length() - len, t.length()).toCharArray();
- * }
- */
- char[] GetSuffix(int len);
-
- /**
- * The lexer calls this function to indicate that it is done with the stream
- * and hence implementations can free any resources held by this class.
- * Again, the body of this function can be just empty and it will not
- * affect the lexer's operation.
- */
- void Done();
-
-
- void setTabSize(int i);
- int getTabSize();
- boolean getTrackLineColumn();
- void setTrackLineColumn(boolean trackLineColumn);
-}
-/* (filtered)*/
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java
index 34397d215b8..aac680f474a 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.java
@@ -18,6 +18,8 @@ import org.apache.lucene.queryparser.surround.query.DistanceQuery;
import org.apache.lucene.queryparser.surround.query.SrndTermQuery;
import org.apache.lucene.queryparser.surround.query.SrndPrefixQuery;
import org.apache.lucene.queryparser.surround.query.SrndTruncQuery;
+import org.apache.lucene.queryparser.charstream.CharStream;
+import org.apache.lucene.queryparser.charstream.FastCharStream;
/**
* This class is generated by JavaCC. The only method that clients should need
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj
index 787ed16bde4..a91800d527b 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj
@@ -46,6 +46,8 @@ import org.apache.lucene.queryparser.surround.query.DistanceQuery;
import org.apache.lucene.queryparser.surround.query.SrndTermQuery;
import org.apache.lucene.queryparser.surround.query.SrndPrefixQuery;
import org.apache.lucene.queryparser.surround.query.SrndTruncQuery;
+import org.apache.lucene.queryparser.charstream.CharStream;
+import org.apache.lucene.queryparser.charstream.FastCharStream;
/**
* This class is generated by JavaCC. The only method that clients should need
diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParserTokenManager.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParserTokenManager.java
index 0f0d30c13ac..d12f2074387 100644
--- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParserTokenManager.java
+++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/QueryParserTokenManager.java
@@ -1,6 +1,9 @@
/* QueryParserTokenManager.java */
/* Generated By:JavaCC: Do not edit this line. QueryParserTokenManager.java */
package org.apache.lucene.queryparser.surround.parser;
+ import org.apache.lucene.queryparser.charstream.CharStream;
+
+
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java
index 91b3e25314f..2338c1e9bfd 100644
--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java
@@ -37,6 +37,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.charstream.CharStream;
import org.apache.lucene.queryparser.classic.QueryParser.Operator;
import org.apache.lucene.queryparser.flexible.standard.CommonQueryParserConfiguration;
import org.apache.lucene.queryparser.util.QueryParserTestBase;
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
index 1f9fd91e956..e0fd4baf3bc 100644
--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
@@ -45,6 +45,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.charstream.FastCharStream;
import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages;
import org.apache.lucene.queryparser.flexible.core.nodes.FuzzyQueryNode;
@@ -54,7 +55,6 @@ import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessor
import org.apache.lucene.queryparser.flexible.messages.MessageImpl;
import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler;
import org.apache.lucene.queryparser.flexible.standard.nodes.WildcardQueryNode;
-import org.apache.lucene.queryparser.flexible.standard.parser.FastCharStream;
import org.apache.lucene.queryparser.flexible.standard.parser.ParseException;
import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser;
import org.apache.lucene.search.BooleanClause.Occur;
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 97c2ad20362..ebcc7359a63 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -126,6 +126,8 @@ Other Changes
* SOLR-14846: Backup/Restore classes no longer take Optional method parameters and will accept nulls instead. (Mike Drob)
+* LUCENE-9531: Consolidated CharStream and FastCharStream classes: these have been moved from each query parser
+ package to org.apache.lucene.queryparser.charstream (Dawid Weiss).
Bug Fixes
---------------------
diff --git a/solr/core/src/java/org/apache/solr/parser/CharStream.java b/solr/core/src/java/org/apache/solr/parser/CharStream.java
deleted file mode 100644
index 2e295cf2443..00000000000
--- a/solr/core/src/java/org/apache/solr/parser/CharStream.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 7.0 */
-/* JavaCCOptions:STATIC=false,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
-package org.apache.solr.parser;
-
-/**
- * This interface describes a character stream that maintains line and
- * column number positions of the characters. It also has the capability
- * to backup the stream to some extent. An implementation of this
- * interface is used in the TokenManager implementation generated by
- * JavaCCParser.
- *
- * All the methods except backup can be implemented in any fashion. backup
- * needs to be implemented correctly for the correct operation of the lexer.
- * Rest of the methods are all used to get information like line number,
- * column number and the String that constitutes a token and are not used
- * by the lexer. Hence their implementation won't affect the generated lexer's
- * operation.
- */
-
-public
-interface CharStream {
-
- /**
- * Returns the next character from the selected input. The method
- * of selecting the input is the responsibility of the class
- * implementing this interface. Can throw any java.io.IOException.
- */
- char readChar() throws java.io.IOException;
-
- @Deprecated
- /**
- * Returns the column position of the character last read.
- * @deprecated
- * @see #getEndColumn
- */
- int getColumn();
-
- @Deprecated
- /**
- * Returns the line number of the character last read.
- * @deprecated
- * @see #getEndLine
- */
- int getLine();
-
- /**
- * Returns the column number of the last character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getEndColumn();
-
- /**
- * Returns the line number of the last character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getEndLine();
-
- /**
- * Returns the column number of the first character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getBeginColumn();
-
- /**
- * Returns the line number of the first character for current token (being
- * matched after the last call to BeginTOken).
- */
- int getBeginLine();
-
- /**
- * Backs up the input stream by amount steps. Lexer calls this method if it
- * had already read some characters, but could not use them to match a
- * (longer) token. So, they will be used again as the prefix of the next
- * token and it is the implementation's responsibility to do this right.
- */
- void backup(int amount);
-
- /**
- * Returns the next character that marks the beginning of the next token.
- * All characters must remain in the buffer between two successive calls
- * to this method to implement backup correctly.
- */
- char BeginToken() throws java.io.IOException;
-
- /**
- * Returns a string made up of characters from the marked token beginning
- * to the current buffer position. Implementations have the choice of returning
- * anything that they want to. For example, for efficiency, one might decide
- * to just return null, which is a valid implementation.
- */
- String GetImage();
-
- /**
- * Returns an array of characters that make up the suffix of length 'len' for
- * the currently matched token. This is used to build up the matched string
- * for use in actions in the case of MORE. A simple and inefficient
- * implementation of this is as follows :
- *
- * {
- * String t = GetImage();
- * return t.substring(t.length() - len, t.length()).toCharArray();
- * }
- */
- char[] GetSuffix(int len);
-
- /**
- * The lexer calls this function to indicate that it is done with the stream
- * and hence implementations can free any resources held by this class.
- * Again, the body of this function can be just empty and it will not
- * affect the lexer's operation.
- */
- void Done();
-
-
- void setTabSize(int i);
- int getTabSize();
- boolean getTrackLineColumn();
- void setTrackLineColumn(boolean trackLineColumn);
-}
-/* (filtered)*/
diff --git a/solr/core/src/java/org/apache/solr/parser/FastCharStream.java b/solr/core/src/java/org/apache/solr/parser/FastCharStream.java
deleted file mode 100644
index 19ecade0284..00000000000
--- a/solr/core/src/java/org/apache/solr/parser/FastCharStream.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.parser;
-
-import java.io.*;
-
-/** An efficient implementation of JavaCC's CharStream interface. <p>Note that
- * this does not do line-number counting, but instead keeps track of the
- * character position of the token in the input, as required by Lucene's {@link
- * org.apache.lucene.analysis.tokenattributes.OffsetAttribute} API.
- * */
-public final class FastCharStream implements CharStream {
- char[] buffer = null;
-
- int bufferLength = 0; // end of valid chars
- int bufferPosition = 0; // next char to read
-
- int tokenStart = 0; // offset in buffer
- int bufferStart = 0; // position in file of buffer
-
- Reader input; // source of chars
-
- /** Constructs from a Reader. */
- public FastCharStream(Reader r) {
- input = r;
- }
-
- @Override
- public final char readChar() throws IOException {
- if (bufferPosition >= bufferLength)
- refill();
- return buffer[bufferPosition++];
- }
-
- private final void refill() throws IOException {
- int newPosition = bufferLength - tokenStart;
-
- if (tokenStart == 0) { // token won't fit in buffer
- if (buffer == null) { // first time: alloc buffer
- buffer = new char[2048];
- } else if (bufferLength == buffer.length) { // grow buffer
- char[] newBuffer = new char[buffer.length*2];
- System.arraycopy(buffer, 0, newBuffer, 0, bufferLength);
- buffer = newBuffer;
- }
- } else { // shift token to front
- System.arraycopy(buffer, tokenStart, buffer, 0, newPosition);
- }
-
- bufferLength = newPosition; // update state
- bufferPosition = newPosition;
- bufferStart += tokenStart;
- tokenStart = 0;
-
- int charsRead = // fill space in buffer
- input.read(buffer, newPosition, buffer.length-newPosition);
- if (charsRead == -1)
- throw READ_PAST_EOF;
- else
- bufferLength += charsRead;
- }
-
- @Override
- public final char BeginToken() throws IOException {
- tokenStart = bufferPosition;
- return readChar();
- }
-
- /**
- * This Exception is used as a signal rather than an exceptional state.
- */
- private static final IOException READ_PAST_EOF = new IOException("read past eof");
-
- @Override
- public final void backup(int amount) {
- bufferPosition -= amount;
- }
-
- @Override
- public final String GetImage() {
- return new String(buffer, tokenStart, bufferPosition - tokenStart);
- }
-
- @Override
- public final char[] GetSuffix(int len) {
- char[] value = new char[len];
- System.arraycopy(buffer, bufferPosition - len, value, 0, len);
- return value;
- }
-
- @Override
- public final void Done() {
- try {
- input.close();
- } catch (IOException e) {
- }
- }
- @Override
- @Deprecated
- /**
- * Returns the column position of the character last read.
- * @deprecated
- * @see #getEndColumn
- */
- public final int getLine() {
- return 1;
- }
- @Override
- @Deprecated
- /**
- * Returns the column position of the character last read.
- * @deprecated
- * @see #getEndColumn
- */
- public final int getColumn() {
- return bufferStart + bufferPosition;
- }
- @Override
- public final int getEndColumn() {
- return bufferStart + bufferPosition;
- }
- @Override
- public final int getEndLine() {
- return 1;
- }
- @Override
- public final int getBeginColumn() {
- return bufferStart + tokenStart;
- }
- @Override
- public final int getBeginLine() {
- return 1;
- }
-
- @Override
- public void setTabSize(int i) {
- throw new RuntimeException("Tab size not implemented.");
- }
-
- @Override
- public int getTabSize() {
- throw new RuntimeException("Tab size not implemented.");
- }
-
- @Override
- public boolean getTrackLineColumn() {
- return false;
- }
-
- @Override
- public void setTrackLineColumn(boolean trackLineColumn) {
- throw new RuntimeException("Line/Column tracking not implemented.");
- }
-}
diff --git a/solr/core/src/java/org/apache/solr/parser/QueryParser.java b/solr/core/src/java/org/apache/solr/parser/QueryParser.java
index 4293879da94..97be79a18e9 100644
--- a/solr/core/src/java/org/apache/solr/parser/QueryParser.java
+++ b/solr/core/src/java/org/apache/solr/parser/QueryParser.java
@@ -15,6 +15,8 @@ import org.apache.lucene.search.Query;
import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.QParser;
+import org.apache.lucene.queryparser.charstream.CharStream;
+import org.apache.lucene.queryparser.charstream.FastCharStream;
public class QueryParser extends SolrQueryParserBase implements QueryParserConstants {
/** The default operator for parsing queries.
@@ -666,27 +668,6 @@ if (splitOnWhitespace == false) {
finally { jj_save(2, xla); }
}
- private boolean jj_3R_6()
- {
- return false;
- }
-
- private boolean jj_3R_3()
- {
- if (jj_scan_token(TERM)) return true;
- jj_lookingAhead = true;
- jj_semLA = getToken(1).kind == TERM && allowedPostMultiTerm(getToken(2).kind);
- jj_lookingAhead = false;
- if (!jj_semLA || jj_3R_6()) return true;
- Token xsp;
- if (jj_3R_7()) return true;
- while (true) {
- xsp = jj_scanpos;
- if (jj_3R_7()) { jj_scanpos = xsp; break; }
- }
- return false;
- }
-
private boolean jj_3_3()
{
Token xsp;
@@ -730,6 +711,27 @@ if (splitOnWhitespace == false) {
return false;
}
+ private boolean jj_3R_6()
+ {
+ return false;
+ }
+
+ private boolean jj_3R_3()
+ {
+ if (jj_scan_token(TERM)) return true;
+ jj_lookingAhead = true;
+ jj_semLA = getToken(1).kind == TERM && allowedPostMultiTerm(getToken(2).kind);
+ jj_lookingAhead = false;
+ if (!jj_semLA || jj_3R_6()) return true;
+ Token xsp;
+ if (jj_3R_7()) return true;
+ while (true) {
+ xsp = jj_scanpos;
+ if (jj_3R_7()) { jj_scanpos = xsp; break; }
+ }
+ return false;
+ }
+
/** Generated Token Manager. */
public QueryParserTokenManager token_source;
/** Current token. */
diff --git a/solr/core/src/java/org/apache/solr/parser/QueryParser.jj b/solr/core/src/java/org/apache/solr/parser/QueryParser.jj
index 1da42dfd92f..8da3f81d9fe 100644
--- a/solr/core/src/java/org/apache/solr/parser/QueryParser.jj
+++ b/solr/core/src/java/org/apache/solr/parser/QueryParser.jj
@@ -38,6 +38,8 @@ import org.apache.lucene.search.Query;
import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.QParser;
+import org.apache.lucene.queryparser.charstream.CharStream;
+import org.apache.lucene.queryparser.charstream.FastCharStream;
public class QueryParser extends SolrQueryParserBase {
/** The default operator for parsing queries.
diff --git a/solr/core/src/java/org/apache/solr/parser/QueryParserTokenManager.java b/solr/core/src/java/org/apache/solr/parser/QueryParserTokenManager.java
index c717057fae8..323a3ef8709 100644
--- a/solr/core/src/java/org/apache/solr/parser/QueryParserTokenManager.java
+++ b/solr/core/src/java/org/apache/solr/parser/QueryParserTokenManager.java
@@ -1,6 +1,9 @@
/* QueryParserTokenManager.java */
/* Generated By:JavaCC: Do not edit this line. QueryParserTokenManager.java */
package org.apache.solr.parser;
+ import org.apache.lucene.queryparser.charstream.CharStream;
+
+
diff --git a/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java b/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
index c05c7582ca7..1a2795cfbaa 100644
--- a/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
+++ b/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
@@ -66,6 +66,8 @@ import org.apache.solr.search.QParser;
import org.apache.solr.search.QueryUtils;
import org.apache.solr.search.SolrConstantScoreQuery;
import org.apache.solr.search.SyntaxError;
+import org.apache.lucene.queryparser.charstream.CharStream;
+import org.apache.lucene.queryparser.charstream.FastCharStream;
import static org.apache.solr.parser.SolrQueryParserBase.SynonymQueryStyle.AS_SAME_TERM;
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
index 447b8303967..e4420558d77 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
@@ -26,12 +26,12 @@ import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.queryparser.charstream.CharStream;
import org.apache.lucene.search.AutomatonQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations;
import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.parser.CharStream;
import org.apache.solr.parser.ParseException;
import org.apache.solr.parser.SolrQueryParserBase;
import org.apache.solr.request.SolrQueryRequest;
diff --git a/solr/core/src/test/org/apache/solr/parser/SolrQueryParserBaseTest.java b/solr/core/src/test/org/apache/solr/parser/SolrQueryParserBaseTest.java
index 68dd38c10f9..75e11749ba2 100644
--- a/solr/core/src/test/org/apache/solr/parser/SolrQueryParserBaseTest.java
+++ b/solr/core/src/test/org/apache/solr/parser/SolrQueryParserBaseTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.parser;
+import org.apache.lucene.queryparser.charstream.CharStream;
import org.apache.lucene.search.Query;
import org.apache.solr.common.SolrException;
import org.apache.solr.request.SolrQueryRequest;