Ghidra is a software reverse engineering (SRE) framework
GP-6308 Added support for mapped meory blocks in MemoryBytePatternSearcher
ghidragonecfdc46e72c5
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/BulkPatternByteMatcher.java b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/BulkPatternByteMatcher.javaindex 7d53f1f4fa..eb28ba9a5c 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/BulkPatternByteMatcher.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/features/base/memsearch/matcher/BulkPatternByteMatcher.java@@ -41,7 +41,7 @@ public class BulkPatternByteMatcher<T extends BytePattern> implements ByteMatche@Overridepublic Iterable<Match<T>> match(ExtendedByteSequence bytes) {List<Match<T>> matches = new ArrayList<>();- matcher.search(bytes, matches, 0);+ matcher.search(bytes, matches);return matches;}diff --git a/Ghidra/Features/Base/src/main/java/ghidra/program/database/ProgramBuilder.java b/Ghidra/Features/Base/src/main/java/ghidra/program/database/ProgramBuilder.javaindex 23add48ab5..bc9a52c7ed 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/program/database/ProgramBuilder.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/program/database/ProgramBuilder.java@@ -316,6 +316,19 @@ public class ProgramBuilder {});}+ public MemoryBlock createMappedMemory(String name, String address, int size,+ String mappedAddress) {+ return tx(() -> {+ Address blockAddress = addr(address);+ Address mapAddress = addr(mappedAddress);+ Memory memory = program.getMemory();+ MemoryBlock block =+ memory.createByteMappedBlock(name, blockAddress, mapAddress, size, false);+ return block;+ });++ }+public MemoryBlock createUninitializedMemory(String name, String address, int size) {return tx(() -> {diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AddressMatch.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AddressMatch.javanew file mode 100644index 0000000000..3f72a776ad--- /dev/null+++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AddressMatch.java@@ -0,0 +1,67 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.util.bytesearch;++import ghidra.program.model.address.Address;++/**+ * Represents a match of a pattern at a given address in program memory.+ *+ * @param <T> The specific implementation of the pattern that was used to create this match+ *+ */+public class AddressMatch<T> extends Match<T> {+ private Address address;++ /**+ * Constructor+ * @param pattern the byte pattern that matched+ * @param offset offset within a searched buffer+ * @param length the length of the matching sequence+ * @param address the address in the program where the match occurred+ */+ public AddressMatch(T pattern, long offset, int length, Address address) {+ super(pattern, offset, length);+ this.address = address;+ }++ /**+ * @return the address where this match occurred+ */+ public Address getAddress() {+ return address;+ }++ @Override+ public String toString() {+ return getPattern().toString() + " @ " + address;+ }++ @Override+ public int hashCode() {+ return super.hashCode() + address.hashCode();+ }++ @Override+ public boolean equals(Object obj) {+ if (!super.equals(obj)) {+ return false;+ }+ AddressMatch<?> other = (AddressMatch<?>) obj;+ return address.equals(other.address);+ }++}diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AddressableByteSequence.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AddressableByteSequence.javaindex ddf28278af..dac1c99424 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AddressableByteSequence.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AddressableByteSequence.java@@ -70,6 +70,21 @@ public class AddressableByteSequence implements ByteSequence {}}+ /**+ * Sets the range of bytes that this object will buffer. This immediately will read the bytes+ * from the byte source into it's internal byte array buffer.+ * @param start the address to start reading bytes+ * @param length the number of bytes to read+ */+ public void setRange(Address start, int length) {+ if (length > capacity) {+ throw new IllegalArgumentException("Length exceeds capacity");+ }+ this.startAddress = start;+ this.length = length;+ byteSource.getBytes(start, bytes, length);+ }+/*** Returns the address of the byte represented by the given index into this buffer.* @param index the index into the buffer to get its associated address@@ -112,14 +127,4 @@ public class AddressableByteSequence implements ByteSequence {public boolean hasAvailableBytes(int index, int length) {return index >= 0 && index + length <= getLength();}-- private void setRange(Address start, int length) {- if (length > capacity) {- throw new IllegalArgumentException("Length exceeds capacity");- }- this.startAddress = start;- this.length = length;- byteSource.getBytes(start, bytes, length);- }-}diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BulkPatternSearcher.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BulkPatternSearcher.javaindex 7debf06a54..77adf93b6f 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BulkPatternSearcher.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BulkPatternSearcher.java@@ -48,6 +48,7 @@ public class BulkPatternSearcher<T extends BytePattern> {private SearchState<T> startState;private int bufferSize = DEFAULT_BUFFER_SIZE;private int uniqueStateCount;+ private int maxPatternLength;/*** Constructor@@ -56,9 +57,18 @@ public class BulkPatternSearcher<T extends BytePattern> {*/public BulkPatternSearcher(List<T> patterns) {this.patterns = patterns;+ maxPatternLength = computeMaxPatternLength();startState = buildStateMachine();}+ private int computeMaxPatternLength() {+ int max = 0;+ for (T pattern : patterns) {+ max = Math.max(max, pattern.getSize());+ }+ return max;+ }+/*** Search the given byte buffer for any of this searcher's patterns.* @param input the byte buffer to search@@ -104,7 +114,7 @@ public class BulkPatternSearcher<T extends BytePattern> {if (nextState == null) {break;}- nextState.addMatchesForCompletedPatterns(results, patternStart);+ nextState.addMatches(results, patternStart);state = nextState;}}@@ -126,7 +136,7 @@ public class BulkPatternSearcher<T extends BytePattern> {if (nextState == null) {break;}- nextState.addMatchesForCompletedPatterns(results, 0);+ nextState.addMatches(results, 0);state = nextState;}}@@ -136,20 +146,24 @@ public class BulkPatternSearcher<T extends BytePattern> {* to the given list of results.* @param bytes the extended byte sequence to search* @param results the list of match results to populate- * @param chunkOffset a constant offset to add to the pattern starts found in this buffer.* Users of this method may have split a larger byte sequence into chunks and the final match* position needs to be the sum of the chunk offset plus the offset within this chunk.*/- public void search(ExtendedByteSequence bytes, List<Match<T>> results, int chunkOffset) {- for (int patternStart = 0; patternStart < bytes.getLength(); ++patternStart) {+ public void search(ExtendedByteSequence bytes, List<Match<T>> results) {+ search(bytes, results, 0);+ }++ private void search(ExtendedByteSequence bytes, List<Match<T>> results, long streamOffset) {+ for (int start = -bytes.getPreLength(); start < bytes.getLength(); start++) {SearchState<T> state = startState;- for (int j = patternStart; j < bytes.getExtendedLength(); j++) {+ for (int j = start; j < bytes.getExtendedLength(); j++) {int index = bytes.getByte(j) & 0xff;SearchState<T> nextState = state.nextStates[index];if (nextState == null) {break;}- nextState.addMatchesForCompletedPatterns(results, patternStart + chunkOffset);+ nextState.addMatchesFilteredByEffectiveStart(results, start, 0,+ bytes.getLength() - 1, streamOffset);state = nextState;}}@@ -182,9 +196,8 @@ public class BulkPatternSearcher<T extends BytePattern> {public void search(InputStream inputStream, long maxRead, List<Match<T>> results,TaskMonitor monitor) throws IOException {RestrictedStream restrictedStream = new RestrictedStream(inputStream, maxRead);- int maxPatternLength = getLongestPatternLength();int bufSize = Math.max(maxPatternLength, bufferSize);- int offset = 0;+ long streamOffset = 0;// The basic strategy is to use two byte buffers and create a virtual buffer with those two// buffers. The first pass will look for patterns that start in the 1st buffer but can@@ -210,9 +223,9 @@ public class BulkPatternSearcher<T extends BytePattern> {ExtendedByteSequence combined =new ExtendedByteSequence(main, pre, post, maxPatternLength);- search(combined, results, offset);+ search(combined, results, streamOffset);monitor.incrementProgress(main.getLength());- offset += main.getLength();+ streamOffset += main.getLength();// rotate buffers and load data into second bufferInputStreamBufferByteSequence tmp = pre;@@ -226,7 +239,7 @@ public class BulkPatternSearcher<T extends BytePattern> {post.load(inputStream, maxPatternLength);ExtendedByteSequence combined =new ExtendedByteSequence(main, pre, post, maxPatternLength);- search(combined, results, offset);+ search(combined, results, streamOffset);monitor.incrementProgress(main.getLength());}@@ -239,6 +252,13 @@ public class BulkPatternSearcher<T extends BytePattern> {this.bufferSize = bufferSize;}+ /**+ * {@return the length of the longest pattern}+ */+ public int getMaxPatternLength() {+ return maxPatternLength;+ }+private SearchState<T> buildStateMachine() {Queue<SearchState<T>> unprocessed = new ArrayDeque<>();@@ -257,14 +277,6 @@ public class BulkPatternSearcher<T extends BytePattern> {return start;}- private int getLongestPatternLength() {- int maxLength = 0;- for (T t : patterns) {- maxLength = Math.max(maxLength, t.getSize());- }- return maxLength;- }-/*** {@return the number of unique states generated. Used for testing.}*/@@ -297,7 +309,7 @@ public class BulkPatternSearcher<T extends BytePattern> {if (state == null) {break;}- state.addMatchesForCompletedPatterns(resultBuffer, patternStart);+ state.addMatches(resultBuffer, patternStart);}patternStart++;}@@ -346,7 +358,7 @@ public class BulkPatternSearcher<T extends BytePattern> {return; // we are a terminal state}for (int inputValue = 0; inputValue < 256; inputValue++) {- List<T> matchedPatterns = getMatchedPatterns(inputValue);+ List<T> matchedPatterns = getMatchingPatternsForTransitionValue(inputValue);if (!matchedPatterns.isEmpty()) {nextStates[inputValue] = getSearchState(matchedPatterns, cache, unresolved);}@@ -393,7 +405,7 @@ public class BulkPatternSearcher<T extends BytePattern> {return newState;}- private List<T> getMatchedPatterns(int inputValue) {+ private List<T> getMatchingPatternsForTransitionValue(int inputValue) {List<T> matchedPatterns = new ArrayList<>();for (T pattern : activePatterns) {if (pattern.isMatch(level, inputValue)) {@@ -403,13 +415,27 @@ public class BulkPatternSearcher<T extends BytePattern> {return matchedPatterns;}- private void addMatchesForCompletedPatterns(Collection<Match<T>> results, int i) {+ private void addMatches(Collection<Match<T>> results, int start) {+ if (completedPatterns == null) {+ return;+ }+ for (T pattern : completedPatterns) {+ results.add(new Match<T>(pattern, start, pattern.getSize()));+ }+ }++ private void addMatchesFilteredByEffectiveStart(Collection<Match<T>> results, int start,+ int min, int max, long streamOffset) {if (completedPatterns == null) {return;}for (T pattern : completedPatterns) {- results.add(new Match<T>(pattern, i, pattern.getSize()));+ int actualStart = start + pattern.getPreSequenceLength();+ if (actualStart >= min && actualStart <= max) {+ results.add(new Match<T>(pattern, streamOffset + start, pattern.getSize()));+ }}+}private List<T> buildFullyMatchedPatternsList() {diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BytePattern.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BytePattern.javaindex 563740cb61..3cc03f9e37 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BytePattern.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/BytePattern.java@@ -16,7 +16,7 @@package ghidra.util.bytesearch;/**- * Interface for Patterns that can be combined into a single state machine that can be+ * Interface for fixed length patterns that can be combined into a single state machine that can be* simultaneously searched for in a byte sequence.*/public interface BytePattern {@@ -35,4 +35,16 @@ public interface BytePattern {*/public boolean isMatch(int patternOffset, int byteValue);+ /**+ * Returns the number of bytes in this pattern that represent a pre-sequence that must match+ * before the official start of the matching pattern. For example if looking for a pattern of+ * "abcd", but only if it follows "xyz", then the pattern would be "xyzabcd", with a pre+ * sequence length of 3. So when this pattern matches, we want the "match to be at the position+ * where the "a" is and not the "x". This is know as "look behind" when using regular+ * expressions.+ * @return the number of bytes in the pattern that represent a required pre sequence before the+ * actual pattern we want to find the position of+ */+ public int getPreSequenceLength();+}diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DittedBitSequence.javaindex f7886a2d71..9a48ef6e6e 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java@@ -482,4 +482,9 @@ public class DittedBitSequence implements BytePattern {}return popcnt;}++ @Override+ public int getPreSequenceLength() {+ return 0;+ }}diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ExtendedByteSequence.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ExtendedByteSequence.javaindex b3f54869e8..3e9f621713 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ExtendedByteSequence.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ExtendedByteSequence.java@@ -75,6 +75,9 @@ public class ExtendedByteSequence implements ByteSequence {return extendedLength;}+ /**+ * {@eturn the length of the pre sequence that is available}+ */public int getPreLength() {return preLength;}@@ -82,7 +85,7 @@ public class ExtendedByteSequence implements ByteSequence {@Overridepublic byte getByte(int i) {if (i < 0) {- return preSequence.getByte(i + preLength);+ return preSequence.getByte(i + preSequence.getLength());}if (i >= mainLength) {return postSequence.getByte(i - mainLength);@@ -96,9 +99,9 @@ public class ExtendedByteSequence implements ByteSequence {throw new IndexOutOfBoundsException();}if (index < 0 && index + size <= 0) {- return preSequence.getBytes(index + preLength, size);+ return preSequence.getBytes(index + preSequence.getLength(), size);}- if (index + size < mainLength) {+ if (index >= 0 && index + size < mainLength) {return mainSequence.getBytes(index, size);}if (index >= mainLength) {diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.javaindex 5a91c69f05..21af3d795d 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java@@ -18,7 +18,7 @@ package ghidra.util.bytesearch;import java.util.Objects;/**- * Represents a match of a BytePattern at a given offset in a byte sequence.+ * Represents a match of a pattern at a given offset in a byte sequence.** @param <T> The specific implementation of the BytePattern that was used to create this match*diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.javaindex 6dd0ca4d34..782839b93f 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java@@ -15,44 +15,44 @@*/package ghidra.util.bytesearch;-import java.io.IOException;-import java.io.InputStream;import java.util.ArrayList;+import java.util.List;-import ghidra.program.model.address.*;+import ghidra.program.model.address.Address;+import ghidra.program.model.address.AddressSetView;import ghidra.program.model.listing.Program;-import ghidra.program.model.mem.MemoryBlock;-import ghidra.util.Msg;import ghidra.util.exception.CancelledException;import ghidra.util.task.TaskMonitor;/**- * Multi pattern/mask/action memory searcher- * Patterns must be supplied/added, or a pre-initialized searchState supplied- *- * Preload search patterns and actions, then call search method.+ * Multi pattern/mask/action memory searcher. This is the legacy memory searcher that specifically+ * uses {@link Pattern} objects which relies on patterns having actions that get invoked as the+ * pattern is found in memory. If you want a simpler, more generic way to search for bulk patterns+ * in memory, you can use the {@link ProgramMemorySearcher}, . If you want an even more generic+ * searcher that isn't restricted to just searching program memory, you can directly use a+ * {@link BulkPatternSearcher}.+ * <P>+ * In this class, patterns can be given at construction time or added one at a time. Optionally,+ * this class can be called with a pre-built BulkPatternSearcher, which is a bit awkward since+ * it is not compatible with adding patterns later. In that case, a new BulkPatternSearcher will be+ * created with only the patterns that were added after construction.+ * <P>+ * Once patterns have been added, simply call the search or searchAll methods to perform a search.*/public class MemoryBytePatternSearcher {- private static final long RESTRICTED_PATTERN_BYTE_RANGE = 32;-BulkPatternSearcher<Pattern> patternSearcher = null;-ArrayList<Pattern> patternList;-- private String searchName = "";+ private String searchName = "Searching";private boolean doExecutableBlocksOnly = false; // only search executable blocks- private long numToSearch = 0;- private long numSearched = 0;-/*** Create with pre-created patternList* @param searchName name of search* @param patternList - list of patterns(bytes/mask/action)*/- public MemoryBytePatternSearcher(String searchName, ArrayList<Pattern> patternList) {+ public MemoryBytePatternSearcher(String searchName, List<Pattern> patternList) {this.searchName = searchName;this.patternList = new ArrayList<Pattern>(patternList);}@@ -83,12 +83,25 @@ public class MemoryBytePatternSearcher {*/public void addPattern(Pattern pattern) {patternList.add(pattern);+ patternSearcher = null;}public void setSearchExecutableOnly(boolean doExecutableBlocksOnly) {this.doExecutableBlocksOnly = doExecutableBlocksOnly;}+ /**+ * Search all initialized memory blocks and associated actions on matches+ *+ * @param program to be searched+ * @param monitor allow canceling and reporting of progress+ *+ * @throws CancelledException if canceled+ */+ public void searchAll(Program program, TaskMonitor monitor) throws CancelledException {+ search(program, null, monitor);+ }+/*** Search initialized memory blocks for all patterns(bytes/mask/action).* Call associated action for each pattern matched.@@ -105,167 +118,55 @@ public class MemoryBytePatternSearcher {patternSearcher = new BulkPatternSearcher<>(patternList);}- numToSearch = getNumToSearch(program, searchSet);- monitor.setMessage(searchName + " Search");- monitor.initialize(numToSearch);+ ProgramMemorySearcher<Pattern> searcher =+ new ProgramMemorySearcher<>(searchName + " Search", program, patternSearcher);- MemoryBlock[] blocks = program.getMemory().getBlocks();- for (MemoryBlock block : blocks) {- monitor.setProgress(numSearched);- // check if entire block has anything that is searchable- if (!block.isInitialized()) {- continue;- }- if (doExecutableBlocksOnly && !block.isExecute()) {- continue;- }- if (searchSet != null && !searchSet.isEmpty() &&- !searchSet.intersects(block.getStart(), block.getEnd())) {- continue;- }-- try {- searchBlock(patternSearcher, program, block, searchSet, monitor);- }- catch (IOException e) {- Msg.error(this, "Unable to scan block " + block.getName() + " for " + searchName);- }- numSearched += block.getSize();+ if (searchSet == null) {+ searchSet = program.getMemory().getAllInitializedAddressSet();}- }-- private long getNumToSearch(Program program, AddressSetView searchSet) {- long numAddresses = 0;- MemoryBlock[] blocks = program.getMemory().getBlocks();- for (MemoryBlock block : blocks) {- // check if entire block has anything that is searchable- if (!block.isInitialized()) {- continue;- }- if (doExecutableBlocksOnly && !block.isExecute()) {- continue;- }- if (searchSet != null && !searchSet.isEmpty() &&- !searchSet.intersects(block.getStart(), block.getEnd())) {- continue;- }- numAddresses += block.getSize();+ if (doExecutableBlocksOnly) {+ searchSet = searchSet.intersect(program.getMemory().getExecuteSet());}- return numAddresses;- }- /**- * Search through bytes of a memory block using the finite state machine (BulkPatterMatcher)- * Apply any additional rules for matching patterns.- *- * @param program is the Program being searched- * @param block is the specific block of bytes being searched- *- * @throws IOException exception during read of memory- * @throws CancelledException canceled search- */- private void searchBlock(BulkPatternSearcher<Pattern> searcher, Program program,- MemoryBlock block, AddressSetView restrictSet, TaskMonitor monitor)- throws IOException, CancelledException {+ searcher.search(searchSet, m -> processMatch(program, m), monitor);+ }- // if no restricted set, make restrict set the full block- AddressSet doneSet;- if (restrictSet == null || restrictSet.isEmpty()) {- doneSet = new AddressSet(block.getStart(), block.getEnd());+ private void processMatch(Program program, AddressMatch<Pattern> match) {+ Pattern pattern = match.getPattern();+ Address addr = match.getAddress();+ // the post rules seem to work off the offset were the first pattern matched, not where+ // its mark start is. Since the address is at the mark offset, we want to subtract that+ // back out+ long rawPatternOffset = addr.getOffset() - pattern.getMarkOffset();+ if (!pattern.checkPostRules(rawPatternOffset)) {+ return;}- else {- doneSet = restrictSet.intersectRange(block.getStart(), block.getEnd());- }-- long numInDoneSet = doneSet.getNumAddresses();- long numInBlock = block.getSize();-- Address blockStartAddr = block.getStart();-- // pull each range off the restricted set- long progress = monitor.getProgress();- AddressRangeIterator addressRanges = doneSet.getAddressRanges();- long numDone = 0;- while (addressRanges.hasNext()) {- monitor.checkCancelled();- monitor.setMessage(searchName + " Search");- monitor.setProgress(progress + (long) (numInBlock * ((float) numDone / numInDoneSet)));- AddressRange addressRange = addressRanges.next();- long numAddressesInRange = addressRange.getLength();-- ArrayList<Match<Pattern>> mymatches = new ArrayList<>();-- long streamoffset = blockStartAddr.getOffset();- // Give block a starting/ending point before this address to search- // patterns might start before, since they have a pre-pattern- // TODO: this is dangerous, since pattern might be very big, but the set should be restricted- // normally only when we are searching for more matching patterns that had a postrule that didn't satisfy- // normally the whole memory blocks will get searched.- long blockOffset = addressRange.getMinAddress().subtract(blockStartAddr);- blockOffset = blockOffset - RESTRICTED_PATTERN_BYTE_RANGE;- if (blockOffset <= 0) {- // don't go before the block start- blockOffset = 0;- }-- // compute number of bytes in the range + 1, and don't search more than that.- long maxBlockSearchLength =- addressRange.getMaxAddress().subtract(blockStartAddr) - blockOffset + 1;-- InputStream data = block.getData();- data.skip(blockOffset);-- searcher.search(data, maxBlockSearchLength, mymatches, monitor);- monitor.checkCancelled();-- monitor.setMessage(searchName + " (Examine Matches)");-- // TODO: DANGER there is much offset<-->address calculation here- // should be OK, since they are all relative to the block.- long matchProgress = progress + (long) (numInBlock * ((float) numDone / numInDoneSet));- for (int i = 0; i < mymatches.size(); ++i) {- monitor.checkCancelled();- monitor.setProgress(- matchProgress + (long) (numAddressesInRange * ((float) i / mymatches.size())));- Match<Pattern> match = mymatches.get(i);- Pattern pattern = match.getPattern();- Address addr =- blockStartAddr.add(pattern.getMarkOffset() + match.getStart() + blockOffset);- long totalOffset = streamoffset + blockOffset + match.getStart();- if (!pattern.checkPostRules(totalOffset)) {- continue;- }-- MatchAction[] matchactions = pattern.getMatchActions();- preMatchApply(matchactions, addr);- for (MatchAction matchaction : matchactions) {- matchaction.apply(program, addr, match);- }-- postMatchApply(matchactions, addr);- }-- numDone += numAddressesInRange;+ MatchAction[] matchactions = pattern.getMatchActions();+ preMatchApply(matchactions, addr);+ for (MatchAction matchaction : matchactions) {+ matchaction.apply(program, addr, match);}++ postMatchApply(matchactions, addr);}/**- * Called before any match rules are applied- * @param matchactions actions that matched- * @param addr address of match+ * Called just before any match rules are applied.+ * @param actions the actions from the pattern that matched+ * @param address address of match*/- public void preMatchApply(MatchAction[] matchactions, Address addr) {+ public void preMatchApply(MatchAction[] actions, Address address) {// override if any initialization needs to be done before rule application}/**- * Called after any match rules are applied- * Can use for cross post rule matching state application and cleanup.- * @param matchactions actions that matched- * @param addr adress of match+ * Called just after any match rules are applied.+ * Can be used for cross post rule matching state application and cleanup.+ * @param actions the actions from the pattern that matched+ * @param address the address of match*/- public void postMatchApply(MatchAction[] matchactions, Address addr) {+ public void postMatchApply(MatchAction[] actions, Address address) {// override if any cleanup from rule match application is needed}}diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.javaindex 192ca14efa..fa63f0e83c 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java@@ -243,4 +243,9 @@ public class Pattern extends DittedBitSequence {}return true;}++ @Override+ public int getPreSequenceLength() {+ return markOffset;+ }}diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ProgramMemorySearcher.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ProgramMemorySearcher.javanew file mode 100644index 0000000000..e789b506d2--- /dev/null+++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/ProgramMemorySearcher.java@@ -0,0 +1,198 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.util.bytesearch;++import java.util.ArrayList;+import java.util.List;+import java.util.function.Consumer;++import ghidra.features.base.memsearch.bytesource.ProgramByteSource;+import ghidra.features.base.memsearch.searcher.MemorySearcher;+import ghidra.program.model.address.*;+import ghidra.program.model.listing.Program;+import ghidra.program.model.mem.Memory;+import ghidra.program.model.mem.MemoryBlock;+import ghidra.util.exception.CancelledException;+import ghidra.util.task.TaskMonitor;++/**+ * Class for efficiently searching for one or more patterns in memory. Patterns used by this+ * class can be any class that implements {@link BytePattern}, so clients are free to create+ * their own custom pattern classes.+ * <P>+ * Note: this searcher searches each memory block individually. It intentionally does not find+ * patterns that span memory blocks (even if the memory blocks are adjacent). If you want patterns+ * to span memory blocks, you can use the {@link MemorySearcher} class, which is not block+ * oriented.+ *+ * @param <T> The specific pattern class type+ */+public class ProgramMemorySearcher<T extends BytePattern> {+ private static final int BUF_SIZE = 4096;+ private BulkPatternSearcher<T> patternSearcher;+ private Program program;+ private String name;+ private int maxPatternLength;++ private AddressableByteSequence pre;+ private AddressableByteSequence main;+ private AddressableByteSequence post;+ private List<Match<T>> intermediateResults = new ArrayList<>();++ /**+ * Constructor+ * @param name the name of the searcher. (Used by the task monitor messages)+ * @param program The program whose memory is to be searched+ * @param patterns the list of pattern objects to search for+ */+ public ProgramMemorySearcher(String name, Program program, List<T> patterns) {+ this(name, program, new BulkPatternSearcher<T>(patterns));+ }++ /**+ * Constructor+ * @param name the name of the searcher. (Used by the task monitor messages)+ * @param program The program whose memory is to be searched+ * @param patternSearcher the pre-constructed pattern searcher which is state-less and be+ * reused, saving the time of building the state machine for the patterns.+ */+ public ProgramMemorySearcher(String name, Program program,+ BulkPatternSearcher<T> patternSearcher) {+ this.name = name;+ this.program = program;+ this.patternSearcher = patternSearcher;+ this.maxPatternLength = patternSearcher.getMaxPatternLength();+ ProgramByteSource programByteSource = new ProgramByteSource(program);+ pre = new AddressableByteSequence(programByteSource, BUF_SIZE);+ main = new AddressableByteSequence(programByteSource, BUF_SIZE);+ post = new AddressableByteSequence(programByteSource, BUF_SIZE);+ }++ /**+ * Searches all initialized memory in the program for the patterns given to this searcher.+ * @param consumer the consumer to be called back when a match is found+ * @param monitor the task monitor for reporting progress and allowing for cancellation+ * @throws CancelledException thrown if the search is cancelled via the task monitor+ */+ public void searchAll(Consumer<AddressMatch<T>> consumer, TaskMonitor monitor)+ throws CancelledException {+ search(program.getMemory().getLoadedAndInitializedAddressSet(), consumer, monitor);+ }++ /**+ * Searches the given address set within initialized memory for the patterns given to this+ * searcher.+ * @param addresses The address within the program to search. This address set will be further+ * restricted to initialized program memory+ * @param consumer the consumer to be called back when a match is found+ * @param monitor the task monitor for reporting progress and allowing for cancellation+ * @throws CancelledException thrown if the search is cancelled via the task monitor+ */+ public void search(AddressSetView addresses, Consumer<AddressMatch<T>> consumer,+ TaskMonitor monitor) throws CancelledException {++ Memory memory = program.getMemory();++ // we can't search in uninitialized memory, so exclude those addresses+ AddressSet initializedAddresses = addresses.intersect(memory.getAllInitializedAddressSet());+ monitor.setMessage(name);+ monitor.initialize(initializedAddresses.getNumAddresses());++ MemoryBlock[] blocks = memory.getBlocks();+ for (MemoryBlock memoryBlock : blocks) {+ monitor.checkCancelled();+ searchBlock(memoryBlock, initializedAddresses, consumer, monitor);+ }+ }++ private void searchBlock(MemoryBlock block, AddressSet addresses,+ Consumer<AddressMatch<T>> consumer, TaskMonitor monitor) throws CancelledException {++ AddressSet blockSet = addresses.intersectRange(block.getStart(), block.getEnd());+ for (AddressRange range : blockSet) {+ searchRange(block, range, consumer, monitor);+ }+ }++ private void searchRange(MemoryBlock block, AddressRange range,+ Consumer<AddressMatch<T>> consumer, TaskMonitor monitor) throws CancelledException {+ pre.clear();+ main.clear();+ post.clear();++ // load data before range to allow for pre sequence patterns to match+ populatePreSequenceForLookBehindPatterns(block, range);++ AddressRangeSplitter splitter = new AddressRangeSplitter(range, BUF_SIZE, true);+ main.setRange(splitter.next());+ while (splitter.hasNext()) {+ monitor.checkCancelled();+ post.setRange(splitter.next());+ performSearch(consumer);+ monitor.incrementProgress(range.getLength());+ rotateBuffers();+ }+ // load data some past end of range to allow pattern to complete+ populatePostSequenceForPatternCompletion(block, range);+ performSearch(consumer);+ }++ private void performSearch(Consumer<AddressMatch<T>> consumer) {+ intermediateResults.clear();+ int overlapSize = maxPatternLength; // number of bytes in pre/post that need to be used+ ExtendedByteSequence sequence = new ExtendedByteSequence(main, pre, post, overlapSize);+ patternSearcher.search(sequence, intermediateResults);+ for (Match<T> match : intermediateResults) {+ long start = match.getStart() + match.getPattern().getPreSequenceLength();+ Address address = main.getAddress((int) start);+ int length = match.getLength();+ T pattern = match.getPattern();+ AddressMatch<T> addressMatch = new AddressMatch<>(pattern, start, length, address);+ consumer.accept(addressMatch);+ }+ }++ private void rotateBuffers() {+ AddressableByteSequence tmp = pre;+ pre = main;+ main = post;+ post = tmp;+ }++ private void populatePreSequenceForLookBehindPatterns(MemoryBlock block, AddressRange range) {+ Address blockStart = block.getStart();+ Address rangeStart = range.getMinAddress();+ if (rangeStart.equals(blockStart)) {+ return;+ }+ // We don't go back beyond block start for pre bytes+ int availablePreBytes = (int) rangeStart.subtract(blockStart);+ int preSize = Math.min(availablePreBytes, maxPatternLength);+ Address preStartAddress = rangeStart.subtract(preSize);+ pre.setRange(preStartAddress, preSize);+ }++ private void populatePostSequenceForPatternCompletion(MemoryBlock block, AddressRange range) {+ Address blockEnd = block.getEnd();+ Address rangeEnd = range.getMaxAddress();+ if (rangeEnd.equals(blockEnd)) {+ return;+ }+ int availablePostBytes = (int) blockEnd.subtract(rangeEnd);+ int postSize = Math.min(availablePostBytes, maxPatternLength);+ post.setRange(rangeEnd.next(), postSize);+ }+}diff --git a/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequence.java b/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequence.javaindex 6dcd80b118..c08eac7996 100644--- a/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequence.java+++ b/Ghidra/Features/Base/src/test/java/ghidra/features/base/memsearch/bytesequence/ByteArrayByteSequence.java@@ -25,6 +25,10 @@ public class ByteArrayByteSequence implements ByteSequence {this.bytes = bytes;}+ public ByteArrayByteSequence(String data) {+ this(data.getBytes());+ }+@Overridepublic int getLength() {return bytes.length;diff --git a/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/BulkPatternSearcherTest.java b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/BulkPatternSearcherTest.javaindex b31f7ab7b0..bd890ac31b 100644--- a/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/BulkPatternSearcherTest.java+++ b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/BulkPatternSearcherTest.java@@ -23,6 +23,7 @@ import java.util.*;import org.junit.Before;import org.junit.Test;+import ghidra.features.base.memsearch.bytesequence.ByteArrayByteSequence;import ghidra.util.task.TaskMonitor;public class BulkPatternSearcherTest {@@ -35,6 +36,7 @@ public class BulkPatternSearcherTest {private TestPattern bcc = new TestPattern("bcc");private BulkPatternSearcher<TestPattern> searcher;+ private List<Match<TestPattern>> results = new ArrayList<Match<TestPattern>>();@Beforepublic void setUp() {@@ -44,7 +46,6 @@ public class BulkPatternSearcherTest {@Testpublic void testMatchWithIntoList() {- List<Match<TestPattern>> results = new ArrayList<Match<TestPattern>>();searcher.search(data.getBytes(), results);Iterator<Match<TestPattern>> it = results.iterator();assertTrue(it.hasNext());@@ -64,7 +65,6 @@ public class BulkPatternSearcherTest {@Testpublic void testMatchWithIntoListWithBufferLimit() {- List<Match<TestPattern>> results = new ArrayList<Match<TestPattern>>();searcher.search(data.getBytes(), 5, results);Iterator<Match<TestPattern>> it = results.iterator();assertTrue(it.hasNext());@@ -113,7 +113,6 @@ public class BulkPatternSearcherTest {String input = "This is a test of the input stream";InputStream is = new ByteArrayInputStream(input.getBytes());- List<Match<TestPattern>> results = new ArrayList<>();Matcher.search(is, results, TaskMonitor.DUMMY);assertEquals(3, results.size());@@ -132,7 +131,6 @@ public class BulkPatternSearcherTest {String input = "This is a test of the input stream";InputStream is = new ByteArrayInputStream(input.getBytes());- List<Match<TestPattern>> results = new ArrayList<>();matcher.search(is, -1, results, TaskMonitor.DUMMY);assertEquals(3, results.size());@@ -152,7 +150,6 @@ public class BulkPatternSearcherTest {String input = "This is a test of the input stream";InputStream is = new ByteArrayInputStream(input.getBytes());- List<Match<TestPattern>> results = new ArrayList<>();matcher.search(is, 24, results, TaskMonitor.DUMMY);assertEquals(2, results.size());@@ -219,7 +216,6 @@ public class BulkPatternSearcherTest {@Testpublic void testSearchBeginningOnly() {- List<Match<TestPattern>> results = new ArrayList<Match<TestPattern>>();searcher.matches(data.getBytes(), data.length(), results);Iterator<Match<TestPattern>> it = results.iterator();assertTrue(it.hasNext());@@ -228,6 +224,68 @@ public class BulkPatternSearcherTest {assertFalse(it.hasNext());}+ @Test+ public void testByteSequenceStartsInMainEndsInPost() {+ TestPattern p = new TestPattern("joebob");+ search("xxxxjoexbob", "xxxjoe", "bob", p);+ assertEquals(1, results.size());+ assertMatch(results.get(0), p, 8);+ }++ @Test+ public void testPreSequencePatterns_patternStartsInPre_effectivelyStartInPre() {+ TestPattern p = new TestPattern("joe", "bob");++ // pre-pattern and effective start are both in pre sequence, so no match+ search("xxjoeb", "obxxx", "xxxx", p);+ assertTrue(results.isEmpty());+ }++ @Test+ public void testPreSequencePatterns_patternStartInPre_effectivelyStartsInMain() {+ TestPattern p = new TestPattern("joe", "bob");++ // pre-patterns starts in pre sequence, effective match start is in main, so this+ // is a match+ search("xxxxjoe", "bobxxx", "xxxx", p);+ assertEquals(1, results.size());+ assertMatch(results.get(0), p, -3);+ }++ @Test+ public void testPreSequencePatterns_patternStartsInMainEndsInPost() {+ TestPattern p = new TestPattern("joe", "bob");++ // create input such that pre-sequence and main sequence start in main, but pattern+ // ends in post sequence, so this is a match+ search("xxx", "xxjoeb", "obxx", p);+ assertEquals(1, results.size());+ assertMatch(results.get(0), p, 2);+ }++ @Test+ public void testPreSequencePattern_patternStartsInMain_effectStartInPost() {+ TestPattern p = new TestPattern("joe", "bob");++ // create input such that the pre-sequence starts in the main, but the actual pattern+ // match start is in the post sequence, so this in not a match++ search("xxx", "xxxjoe", "bob", p);+ assertTrue(results.isEmpty());+ }++ private void search(String preData, String mainData, String postData, TestPattern p) {+ ByteSequence pre = new ByteArrayByteSequence(preData);+ ByteSequence main = new ByteArrayByteSequence(mainData);+ ByteSequence post = new ByteArrayByteSequence(postData);+ ExtendedByteSequence sequence = new ExtendedByteSequence(main, pre, post, 10);++ BulkPatternSearcher<TestPattern> patternSearcher =+ new BulkPatternSearcher<TestPattern>(List.of(p));+ patternSearcher.search(sequence, results);++ }+private void assertMatch(Match<TestPattern> match, TestPattern expectedPattern, int start) {assertEquals(new Match<>(expectedPattern, start, expectedPattern.getSize()), match);}@@ -235,10 +293,16 @@ public class BulkPatternSearcherTest {private class TestPattern extends DittedBitSequence {private String inputString;+ private int preSequenceLength;public TestPattern(String inputString) {- super(getBytes(inputString), getMask(inputString));- this.inputString = inputString;+ this("", inputString);+ }++ public TestPattern(String preSequence, String matchSequence) {+ super(getBytes(preSequence + matchSequence), getMask(preSequence + matchSequence));+ this.inputString = preSequence + matchSequence;+ this.preSequenceLength = preSequence.length();}private static byte[] getMask(String inputString) {@@ -269,5 +333,10 @@ public class BulkPatternSearcherTest {return inputString;}+ @Override+ public int getPreSequenceLength() {+ return preSequenceLength;+ }+}}diff --git a/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/MemoryBytePatternSearcherTest.java b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/MemoryBytePatternSearcherTest.javanew file mode 100644index 0000000000..d6a78d8a9d--- /dev/null+++ b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/MemoryBytePatternSearcherTest.java@@ -0,0 +1,285 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.util.bytesearch;++import static org.junit.Assert.*;++import java.util.*;++import org.junit.Before;+import org.junit.Test;++import ghidra.program.database.ProgramBuilder;+import ghidra.program.database.ProgramDB;+import ghidra.program.model.address.Address;+import ghidra.program.model.address.AddressSet;+import ghidra.program.model.data.CategoryPath;+import ghidra.program.model.listing.Program;+import ghidra.program.model.mem.MemoryBlock;+import ghidra.test.AbstractGhidraHeadlessIntegrationTest;+import ghidra.util.task.TaskMonitor;+import ghidra.xml.XmlPullParser;++public class MemoryBytePatternSearcherTest extends AbstractGhidraHeadlessIntegrationTest {++ private ProgramBuilder builder;+ private ProgramDB program;+ List<AddressMatch<Pattern>> results = new ArrayList<>();+ private MatchAction[] matchActions;++ @Before+ public void setup() throws Exception {+ program = buildProgram();+ matchActions = new MatchAction[1];+ matchActions[0] = new TestMatchAction();+ }++ @Test+ public void testMatchesDontSpanBlocks() throws Exception {+ builder.setString("0x1001010", "Hello");+ builder.setString("0x10010ff", "Hello"); // crosses block boundary, shouldn't be found+ builder.setString("0x1001520", "There");+ TestPattern p1 = new TestPattern("Hello");+ TestPattern p2 = new TestPattern("There");+ MemoryBytePatternSearcher searcher = createSearcher(p1, p2);++ searcher.search(program, null, TaskMonitor.DUMMY);++ assertEquals(2, results.size());+ assertMatch(results.get(0), addr(0x1001010));+ assertMatch(results.get(1), addr(0x1001520));+ }++ @Test+ public void testMatchInAddressSet() throws Exception {+ builder.setString("0x1001010", "Hello");+ TestPattern p1 = new TestPattern("Hello");+ MemoryBytePatternSearcher searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001005, 0x1001030);++ searcher.search(program, searchSet, TaskMonitor.DUMMY);++ assertEquals(1, results.size());+ assertMatch(results.get(0), addr(0x1001010));+ }++ @Test+ public void testMatchStartsOutsideRange() throws Exception {+ builder.setString("0x1001010", "Hello");+ TestPattern p1 = new TestPattern("Hello");+ MemoryBytePatternSearcher searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001011, 0x1001030);++ searcher.search(program, searchSet, TaskMonitor.DUMMY);++ assertTrue(results.isEmpty());+ }++ @Test+ public void testMatchStartsInRangeAndExtendsOut() throws Exception {+ builder.setString("0x1001010", "Hello");+ TestPattern p1 = new TestPattern("Hello");+ MemoryBytePatternSearcher searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001010, 0x1001012);++ searcher.search(program, searchSet, TaskMonitor.DUMMY);++ assertEquals(1, results.size());+ assertMatch(results.get(0), addr(0x1001010));+ }++ @Test+ public void testPreSequencePattern() throws Exception {+ builder.setString("0x1001010", "HelloThere");+ TestPattern p1 = new TestPattern("HelloThere", 5);+ MemoryBytePatternSearcher searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001000, 0x1001040);++ searcher.search(program, searchSet, TaskMonitor.DUMMY);+ assertEquals(1, results.size());+ assertMatch(results.get(0), addr(0x1001015));+ }++ @Test+ public void testPreSequencePattern_preStartsBeforeRange() throws Exception {+ builder.setString("0x1001010", "HelloThere");+ TestPattern p1 = new TestPattern("HelloThere", 5);+ MemoryBytePatternSearcher searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001012, 0x1001040);++ searcher.search(program, searchSet, TaskMonitor.DUMMY);+ assertEquals(1, results.size());+ assertMatch(results.get(0), addr(0x1001015));+ }++ @Test+ public void testPreSequencePattern_primaryStartsBeforeRange() throws Exception {+ builder.setString("0x1001010", "HelloThere");+ TestPattern p1 = new TestPattern("HelloThere", 5);+ MemoryBytePatternSearcher searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001016, 0x1001040);++ searcher.search(program, searchSet, TaskMonitor.DUMMY);+ assertTrue(results.isEmpty());+ }++ @Test+ public void testSearchExecuteBlocksOnly() throws Exception {+ MemoryBlock block = builder.createMemory("execute", "0x2000000", 0x100);+ builder.setExecute(block, true);++ builder.setString("0x1001010", "Hello");+ builder.setString("0x2000010", "Hello");+ TestPattern p1 = new TestPattern("Hello");+ MemoryBytePatternSearcher searcher = createSearcher(p1);+ searcher.setSearchExecutableOnly(true);+ searcher.searchAll(program, TaskMonitor.DUMMY);++ assertEquals(1, results.size());+ assertMatch(results.get(0), addr(0x2000010));+ }++ @Test+ public void testAddPatternsLater() throws Exception {+ builder.setString("0x1001010", "Hello");+ builder.setString("0x1001030", "There");+ TestPattern p1 = new TestPattern("Hello");+ MemoryBytePatternSearcher searcher = createSearcher(p1);+ searcher.searchAll(program, TaskMonitor.DUMMY);++ assertEquals(1, results.size());+ assertMatch(results.get(0), addr(0x1001010));++ results.clear();+ searcher.addPattern(new TestPattern("There"));+ searcher.searchAll(program, TaskMonitor.DUMMY);+ assertEquals(2, results.size());+ assertMatch(results.get(0), addr(0x1001010));+ assertMatch(results.get(1), addr(0x1001030));+ }++ @Test+ public void testMappedBlocks() throws Exception {+ builder.createMappedMemory("mapped", "0x2000000", 0x10000, "0x1000000");++ builder.setString("0x1001100", "Hello");+ builder.setString("0x1001510", "Hello");+ TestPattern p1 = new TestPattern("Hello");+ MemoryBytePatternSearcher searcher = createSearcher(p1);+ searcher.searchAll(program, TaskMonitor.DUMMY);+ assertEquals(4, results.size());+ assertMatch(results.get(0), addr(0x1001100));+ assertMatch(results.get(1), addr(0x1001510));+ assertMatch(results.get(2), addr(0x2001100));+ assertMatch(results.get(3), addr(0x2001510));++ }++ private Address addr(long offset) {+ return builder.addr(offset);+ }++ private AddressSet addrSet(long start, long end) {+ return new AddressSet(addr(start), addr(end));+ }++ private ProgramDB buildProgram() throws Exception {+ builder = new ProgramBuilder("Program1", ProgramBuilder._TOY, this);++ builder.createMemory("b1", "0x1001000", 0x100);+ builder.createMemory("b2", "0x1001100", 0x100);+ builder.createMemory("b3", "0x1001500", 0x100);+ CategoryPath miscPath = new CategoryPath("/MISC");+ builder.addCategory(miscPath);+ return builder.getProgram();+ }++ private MemoryBytePatternSearcher createSearcher(Pattern... patterns) {+ List<Pattern> patternList = Arrays.asList(patterns);+ return new MemoryBytePatternSearcher("Test", patternList);+ }++ private void assertMatch(AddressMatch<Pattern> addressMatch, Address addr) {+ assertEquals("Match address", addr, addressMatch.getAddress());+ }++ private class TestPattern extends Pattern {++ private String inputString;+ private int preSequenceLength;++ public TestPattern(String inputString) {+ this(inputString, 0);+ }++ public TestPattern(String inputString, int preSequenceLength) {+ super(new DittedBitSequence(getBytes(inputString), getMask(inputString)),+ preSequenceLength, new PostRule[0], matchActions);+ this.inputString = inputString;+ this.preSequenceLength = preSequenceLength;+ }++ private static byte[] getMask(String inputString) {+ byte[] mask = new byte[inputString.length()];+ for (int i = 0; i < inputString.length(); i++) {+ if (inputString.charAt(i) == '.') {+ mask[i] = 0;+ }+ else {+ mask[i] = (byte) 0xff;+ }+ }+ return mask;+ }++ private static byte[] getBytes(String inputString) {+ byte[] bytes = inputString.getBytes();+ for (int i = 0; i < inputString.length(); i++) {+ if (inputString.charAt(i) == '.') {+ bytes[i] = 0;+ }+ }+ return bytes;+ }++ @Override+ public String toString() {+ return inputString;+ }++ @Override+ public int getPreSequenceLength() {+ return preSequenceLength;+ }++ }++ private class TestMatchAction implements MatchAction {++ @Override+ public void apply(Program program, Address addr, Match<Pattern> match) {+ results.add(new AddressMatch<Pattern>(match.getPattern(), match.getStart(),+ match.getLength(), addr));+ }++ @Override+ public void restoreXml(XmlPullParser parser) {+ // not used+ }++ }++}diff --git a/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/ProgramMemorySearcherTest.java b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/ProgramMemorySearcherTest.javanew file mode 100644index 0000000000..6040408a64--- /dev/null+++ b/Ghidra/Features/Base/src/test/java/ghidra/util/bytesearch/ProgramMemorySearcherTest.java@@ -0,0 +1,230 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.util.bytesearch;++import static org.junit.Assert.*;++import java.util.*;++import org.junit.Before;+import org.junit.Test;++import ghidra.program.database.ProgramBuilder;+import ghidra.program.database.ProgramDB;+import ghidra.program.model.address.Address;+import ghidra.program.model.address.AddressSet;+import ghidra.program.model.data.CategoryPath;+import ghidra.test.AbstractGhidraHeadlessIntegrationTest;+import ghidra.util.task.TaskMonitor;++public class ProgramMemorySearcherTest extends AbstractGhidraHeadlessIntegrationTest {++ private ProgramBuilder builder;+ private ProgramDB program;+ List<AddressMatch<TestPattern>> results = new ArrayList<>();++ @Before+ public void setup() throws Exception {+ program = buildProgram();+ }++ @Test+ public void testMatchesDontSpanBlocks() throws Exception {+ builder.setString("0x1001010", "Hello");+ builder.setString("0x10010ff", "Hello"); // crosses block boundary, shouldn't be found+ builder.setString("0x1001520", "There");+ TestPattern p1 = new TestPattern("Hello");+ TestPattern p2 = new TestPattern("There");+ ProgramMemorySearcher<TestPattern> searcher = createSearcher(p1, p2);++ searcher.searchAll(m -> results.add(m), TaskMonitor.DUMMY);++ assertEquals(2, results.size());+ assertMatch(results.get(0), addr(0x1001010));+ assertMatch(results.get(1), addr(0x1001520));+ }++ @Test+ public void testMatchInAddressSet() throws Exception {+ builder.setString("0x1001010", "Hello");+ TestPattern p1 = new TestPattern("Hello");+ ProgramMemorySearcher<TestPattern> searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001005, 0x1001030);++ searcher.search(searchSet, m -> results.add(m), TaskMonitor.DUMMY);++ assertEquals(1, results.size());+ assertMatch(results.get(0), addr(0x1001010));+ }++ @Test+ public void testMatchStartsOutsideRange() throws Exception {+ builder.setString("0x1001010", "Hello");+ TestPattern p1 = new TestPattern("Hello");+ ProgramMemorySearcher<TestPattern> searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001011, 0x1001030);++ searcher.search(searchSet, m -> results.add(m), TaskMonitor.DUMMY);++ assertTrue(results.isEmpty());+ }++ @Test+ public void testMatchStartsInRangeAndExtendsOut() throws Exception {+ builder.setString("0x1001010", "Hello");+ TestPattern p1 = new TestPattern("Hello");+ ProgramMemorySearcher<TestPattern> searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001010, 0x1001012);++ searcher.search(searchSet, m -> results.add(m), TaskMonitor.DUMMY);++ assertEquals(1, results.size());+ assertMatch(results.get(0), addr(0x1001010));+ }++ @Test+ public void testPreSequencePattern() throws Exception {+ builder.setString("0x1001010", "HelloThere");+ TestPattern p1 = new TestPattern("HelloThere", 5);+ ProgramMemorySearcher<TestPattern> searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001000, 0x1001040);++ searcher.search(searchSet, m -> results.add(m), TaskMonitor.DUMMY);+ assertEquals(1, results.size());+ assertMatch(results.get(0), addr(0x1001015));+ }++ @Test+ public void testPreSequencePattern_preStartsBeforeRange() throws Exception {+ builder.setString("0x1001010", "HelloThere");+ TestPattern p1 = new TestPattern("HelloThere", 5);+ ProgramMemorySearcher<TestPattern> searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001012, 0x1001040);++ searcher.search(searchSet, m -> results.add(m), TaskMonitor.DUMMY);+ assertEquals(1, results.size());+ assertMatch(results.get(0), addr(0x1001015));+ }++ @Test+ public void testPreSequencePattern_primaryStartsBeforeRange() throws Exception {+ builder.setString("0x1001010", "HelloThere");+ TestPattern p1 = new TestPattern("HelloThere", 5);+ ProgramMemorySearcher<TestPattern> searcher = createSearcher(p1);+ AddressSet searchSet = addrSet(0x1001016, 0x1001040);++ searcher.search(searchSet, m -> results.add(m), TaskMonitor.DUMMY);+ assertTrue(results.isEmpty());+ }++ @Test+ public void testBigBlock() throws Exception {+ builder.createMemory("big", "0x2000000", 0x10000);+ builder.setString("0x2000000", "Hello");+ builder.setString("0x2123456", "Hello");+ builder.setString("0x2fffff0", "Hello");+ TestPattern p1 = new TestPattern("Hello");+ ProgramMemorySearcher<TestPattern> searcher = createSearcher(p1);++ searcher.searchAll(m -> results.add(m), TaskMonitor.DUMMY);+ assertEquals(3, results.size());+ assertMatch(results.get(0), addr(0x2000000));+ assertMatch(results.get(1), addr(0x2123456));+ assertMatch(results.get(2), addr(0x2fffff0));++ }++ private Address addr(long offset) {+ return builder.addr(offset);+ }++ private AddressSet addrSet(long start, long end) {+ return new AddressSet(addr(start), addr(end));+ }++ private ProgramDB buildProgram() throws Exception {+ builder = new ProgramBuilder("Program1", ProgramBuilder._TOY, this);++ builder.createMemory("b1", "0x1001000", 0x100);+ builder.createMemory("b2", "0x1001100", 0x100);+ builder.createMemory("b3", "0x1001500", 0x100);+ CategoryPath miscPath = new CategoryPath("/MISC");+ builder.addCategory(miscPath);+ return builder.getProgram();+ }++ private ProgramMemorySearcher<TestPattern> createSearcher(TestPattern... patterns) {+ List<TestPattern> patternList = Arrays.asList(patterns);+ ProgramMemorySearcher<TestPattern> searcher =+ new ProgramMemorySearcher<TestPattern>("Test", program, patternList);+ return searcher;+ }++ private void assertMatch(AddressMatch<TestPattern> addressMatch, Address addr) {+ assertEquals("Match address", addr, addressMatch.getAddress());+ }++ private class TestPattern extends DittedBitSequence {++ private String inputString;+ private int preSequenceLength;++ public TestPattern(String inputString) {+ this(inputString, 0);+ }++ public TestPattern(String inputString, int preSequenceLength) {+ super(getBytes(inputString), getMask(inputString));+ this.inputString = inputString;+ this.preSequenceLength = preSequenceLength;+ }++ private static byte[] getMask(String inputString) {+ byte[] mask = new byte[inputString.length()];+ for (int i = 0; i < inputString.length(); i++) {+ if (inputString.charAt(i) == '.') {+ mask[i] = 0;+ }+ else {+ mask[i] = (byte) 0xff;+ }+ }+ return mask;+ }++ private static byte[] getBytes(String inputString) {+ byte[] bytes = inputString.getBytes();+ for (int i = 0; i < inputString.length(); i++) {+ if (inputString.charAt(i) == '.') {+ bytes[i] = 0;+ }+ }+ return bytes;+ }++ @Override+ public String toString() {+ return inputString;+ }++ @Override+ public int getPreSequenceLength() {+ return preSequenceLength;+ }++ }++}