Ghidra is a software reverse engineering (SRE) framework
GP-6921: from GP-X
d-millare1dddb3ec0dd
diff --git a/Ghidra/Features/BSim/certification.manifest b/Ghidra/Features/BSim/certification.manifestindex 951b1874fb..480af25ac1 100755--- a/Ghidra/Features/BSim/certification.manifest+++ b/Ghidra/Features/BSim/certification.manifest@@ -41,6 +41,7 @@ src/main/help/help/topics/BSimSearchPlugin/images/BSimOverviewResults.png||GHIDRsrc/main/help/help/topics/BSimSearchPlugin/images/BSimResultsProvider.png||GHIDRA||||END|src/main/help/help/topics/BSimSearchPlugin/images/BSimSearchDialog.png||GHIDRA||||END|src/main/help/help/topics/BSimSearchPlugin/images/ManageServersDialog.png||GHIDRA||||END|+src/main/help/help/topics/StructureRecovery/StructureRecovery.html||GHIDRA||||END|src/main/resources/bsim.log4j.xml||GHIDRA||||END|src/main/resources/images/checkmark_yellow.gif||GHIDRA||||END|src/main/resources/images/connected.png||GHIDRA||||END|diff --git a/Ghidra/Features/BSim/src/main/help/help/TOC_Source.xml b/Ghidra/Features/BSim/src/main/help/help/TOC_Source.xmlindex ba54898ec6..9f8be0f64d 100755--- a/Ghidra/Features/BSim/src/main/help/help/TOC_Source.xml+++ b/Ghidra/Features/BSim/src/main/help/help/TOC_Source.xml@@ -170,6 +170,28 @@text="BSim Command (bsim)"target="help/topics/BSim/CommandLineReference.html#BSimCommand"/></tocdef>++ <tocdef id="StructureRecovery" sortgroup="f"+ text="Structure Recovery Using BSim"+ target="help/topics/StructureRecovery/StructureRecovery.html" >++ <tocdef id="TagStructureUse"+ sortgroup="a"+ text="Tag structure use"+ target="help/topics/StructureRecovery/StructureRecovery.html#TagStructureUse"/>+ <tocdef id="MatchFunctions"+ sortgroup="b"+ text="Match functions"+ target="help/topics/StructureRecovery/StructureRecovery.html#MatchFunctions"/>+ <tocdef id="GenerateStructure"+ sortgroup="c"+ text="Generate structure"+ target="help/topics/StructureRecovery/StructureRecovery.html#GenerateStructure"/>+ <tocdef id="Options"+ sortgroup="z"+ text="Options"+ target="help/topics/StructureRecovery/StructureRecovery.html#Options"/>+ </tocdef></tocdef></tocref></tocroot>diff --git a/Ghidra/Features/BSim/src/main/help/help/topics/StructureRecovery/StructureRecovery.html b/Ghidra/Features/BSim/src/main/help/help/topics/StructureRecovery/StructureRecovery.htmlnew file mode 100644index 0000000000..d7b65437cf--- /dev/null+++ b/Ghidra/Features/BSim/src/main/help/help/topics/StructureRecovery/StructureRecovery.html@@ -0,0 +1,203 @@+<html>+<head>+<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">+<title>Structure Recovery Using BSim</title>+<link rel="stylesheet" type="text/css" href="help/shared/DefaultStyle.css">+<link rel="stylesheet" type="text/css" href="../../shared/languages.css">+<meta name="generator" content="DocBook XSL Stylesheets V1.79.1">+</head>+<body><div class="chapter">+<div class="titlepage"><div><div><h1 class="title">+<a name="StructureRecovery"></a>Structure Recovery Using BSim</h1></div></div></div>++ <p>+ The StructureRecovery plugin allows users to build a structure in a target program based on the+ field uses in a source program. The process involves three steps. First, a source program is+ mined for field references from a known version of the structure to recover the use addresses+ and offsets. Second, functions in the source program bookmarked by field use are matched to+ functions in the target program using the local variation of BSim, which does not require a database.+ Third, BSim is used to create 'pinnings' between the data-flow graphs for each pair of functions+ which identify vertex-vertex correspondences using the decoompiler's diffing algorithm.+ Vertices with addresses from the first step are examined to identify offset information, which+ then becomes input to the construction of a new structure in the target program.+ </p>+ <p>+ The success of this process depends on a number of factors. There are cases, for instance, in+ which nested structures may be missed by the structure use task, and, of course, certain fields+ may not be referenced at all. BSim, here as elsewhere, depends on a number of parameters that+ determine what constitutes a match, which are exposed as options. Unmatched functions will+ potentially result in gaps in the structure. Finally, the pinning comparison attempts to reconcile+ disparate pcode representations, but may fail in more complicated scenarios. When successful,+ the resulting structure will be stored in the target program's DataTypeManager as "RECOVERED_structure_name".+ </p>+ <p>+ Each of the three tasks is relatively expensive. It may be necessary to save results from each+ across sessions. This is done using bookmarks (viewable from the listing or the Bookmarks viewer), which+ capture the structure use and function-to-function matches. The former are represented as+ "USES_structure_name_offset:address"; the latter as "source function id→target function id".+ The function-match bookmarks will be present in both the source and target programs.+ </p>+ <p>+ ALL tasks should run with the source program active.+ </p>++ <div class="section">+ <div class="titlepage"><div><div><h2 class="title" style="clear: both">+ <a name="MainActions"></a>Principal Actions (Tools → Structure Recovery)</h2></div></div></div>++ <div class="sect2">+ <div class="titlepage"><div><div><h3 class="title">+ <a name="TagStructureUse"></a>Tag Structure Use</h3></div></div></div>+ <p>+ Locates all non-zero field uses within a program for a specified structure.+ </p>+ </div>++ <div class="sect2">+ <div class="titlepage"><div><div><h3 class="title">+ <a name="MatchFunctions"></a>Match Functions</h3></div></div></div>+ <p>+ Uses BSim to match functions with structure use in the source program to similar functions+ in a target program.+ </p>+ </div>++ <div class="sect2">+ <div class="titlepage"><div><div><h3 class="title">+ <a name="GenerateStructure"></a>Generate structure</h3></div></div></div>+ <p>+ Compares data-flow 'pinnings' for matched functions and uses the resulting offsets to+ reconstruct an approximate match to the original structure for the target progam.+ </p>+ </div>+</div>+</div>++<div class="section">+ <div class="titlepage"><div><div><h2 class="title" style="clear: both">+ <a name="SecondaryActions"></a>Secondary Actions (Tools → Structure Recovery)</h2></div></div></div>+ <div class="sect2">+ <div class="titlepage"><div><div><h3 class="title">+ <a name="StructureUses"></a>Structure Uses: map to bookmarks</h3></div></div></div>+ <p>+ Converts (persistent) bookmarks containing structure-use information, typically generated+ by the 'Tag Structure Use' action, to a (non-persistent) map and vice-versa.+ </p>+ </div>++ <div class="sect2">+ <div class="titlepage"><div><div><h3 class="title">+ <a name="FunctionMatches"></a>Function Matches: map to bookmarks</h2></div></div></div>++ <p>+ Converts (persistent) bookmarks containing function-function matches, typically generated+ by the 'Match Functions' action, to a (non-persistent) map and vice-versa.+ </p>++ <div class="section">+ <div class="titlepage"><div><div><h2 class="title" style="clear: both">+ <a name="Options"></a>Tool Options (Options/Structure Recovery)</h2></div></div></div>++ <p>+ Options governing various aspects of the structure recovery.+ </p>++ <div class="titlepage"><div><div><h3 class="title">+ <a name="StructureToRecover"></a>Structure to recover</h3></div></div></div>+ <p>+ The full path to the data type to be replicated ('Tag Structure Use' will query if empty).+ </p>+ </div>++ <div class="titlepage"><div><div><h3 class="title">+ <a name="AddStructureUseBookmarks"></a>Add structure-use bookmarks</h3></div></div></div>+ <p>+ Should bookmarks by generated by 'Tag Structure Use'.+ </p>+ </div>++ <div class="sect2">+ <div class="titlepage"><div><div><h3 class="title">+ <a name="AddFunctionMatchBookmarks"></a>Add function-match bookmarks</h3></div></div></div>+ <p>+ Should bookmarks by generated by 'Match Functions'.+ </p>+ </div>++ <div class="sect2">+ <div class="titlepage"><div><div><h3 class="title">+ <a name="MatchConfidenceLowerBound"></a>Match-confidence lower bound</h3></div></div></div>+ <p>+ Used by 'Match Functions' (see BSim documentation for details).+ </p>+ </div>++<div class="sect2">+<div class="titlepage"><div><div><h3 class="title">+<a name="MatchSimilarityLowerBound"></a>Match-similarity lower bound</h3></div></div></div>+ <p>+ Used by 'Match Functions' (see BSim documentation for details).+ </p>+</div>++<div class="sect2">+<div class="titlepage"><div><div><h3 class="title">+<a name="MatchSimilarityUpperBound"></a>Match-similarity upper bound</h3></div></div></div>+ <p>+ Used by 'Match Functions' (see BSim documentation for details).+ </p>+</div>++<div class="sect2">+<div class="titlepage"><div><div><h3 class="title">+<a name="SelfSignificanceBound"></a>Self-significance bound</h3></div></div></div>+ <p>+ Used by 'Match Functions' (see BSim documentation for details).+ </p>+</div>++<div class="sect2">+<div class="titlepage"><div><div><h3 class="title">+<a name="MinConfidence"></a>Min confidence</h3></div></div></div>+ <p>+ Used by 'Match Functions' (see BSim documentation for details).+ </p>+</div>++<div class="sect2">+<div class="titlepage"><div><div><h3 class="title">+<a name="MaxStructureOffset"></a>Max structure offset</h3></div></div></div>+ <p>+ The largest offset considered valid by 'Generate structure' (prevents confusion+ between offsets and addresses).+ </p>+</div>++<div class="sect2">+<div class="titlepage"><div><div><h3 class="title">+<a name="ExcludedOffsets"></a>Excluded offsets (hex)</h3></div></div></div>+ <p>+ Comma-separated hex values to be ignored by 'Generate structure'. Principally used when+ analyzing a randomized structure that amalgamates randomized fields in a common block,+ e.g. 0x40 for the Linux task_struct randomized with symbols.+ </p>+</div>++<div class="sect2">+<div class="titlepage"><div><div><h3 class="title">+<a name="MatchOn"></a>Match on...</h3></div></div></div>+ <p>+ Should the function-function matching rely on solo matches, solo matches plus matches+ exceeding the 'Min confidence', or should a full maximum-weight bipartite match be+ computed.+ </p>+</div>++</div>++</dl></div>+ </div>+++</div></body>+</html>diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/ComparePinningsTask.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/ComparePinningsTask.javanew file mode 100644index 0000000000..5fcdd053da--- /dev/null+++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/ComparePinningsTask.java@@ -0,0 +1,400 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.features.bsim.gui.structs;++import java.util.*;+import java.util.Map.Entry;++import db.Transaction;+import ghidra.app.decompiler.*;+import ghidra.app.services.ConsoleService;+import ghidra.features.codecompare.graphanalysis.DataVertex;+import ghidra.features.codecompare.graphanalysis.Pinning;+import ghidra.program.model.address.AddressSet;+import ghidra.program.model.data.*;+import ghidra.program.model.listing.Function;+import ghidra.program.model.listing.Program;+import ghidra.program.model.pcode.*;+import ghidra.util.Msg;+import ghidra.util.exception.CancelledException;+import ghidra.util.task.Task;+import ghidra.util.task.TaskMonitor;++public class ComparePinningsTask extends Task {++ private StructureRecoveryPlugin plugin;+ private ConsoleService console;++ private static int DECOMPILER_TIMEOUT = 60;++ private final Program currentProgram;+ private final Program targetProgram;+ private DecompInterface currentInterface;+ private DecompInterface targetInterface;++ private Map<Program, Map<Function, DecompileResults>> decompilerResults =+ new HashMap<>();+ private Map<Long, Map<Long, Float>> results = new HashMap<>();++ private Map<String, Long> offsets;+ private Map<String, AddressSet> addressesByField;+ private Map<Function, Set<Function>> functionMap;+ private long maxOffset = 0x100000;++ private record PinningMatch(float weight, int sourceOffset, int targetOffset) {+ @Override+ public String toString() {+ return Integer.toHexString(sourceOffset) + ":" + Integer.toHexString(targetOffset) ++ " (" + weight + ")";+ }+ }++ public ComparePinningsTask(StructureRecoveryPlugin plugin) {+ super("Compare Pinnings", true, false, false);+ this.plugin = plugin;+ this.console = plugin.getConsole();+ this.currentProgram = plugin.getCurrentProgram();+ this.targetProgram = plugin.getTargetProgram();+ this.offsets = plugin.getOffsets();+ this.addressesByField = plugin.getAddressesByField();+ this.functionMap = plugin.getFunctionMap();+ this.maxOffset = plugin.getMaxOffset();+ }++ @Override+ public void run(TaskMonitor monitor) {+ String taskName = getTaskTitle();+ try {+ Thread.currentThread().setName(taskName);+ console.addMessage(taskName, "Running...");+ initDecompiler();+ comparePinnings(monitor);+ console.addMessage(taskName, "Finished!");+ }+ catch (Exception e) {+ if (!monitor.isCancelled()) {+ Msg.showError(this, null, getTaskTitle(), "Error running task: " + taskName ++ "\n" + e.getClass().getName() + ": " + e.getMessage(), e);+ console.addErrorMessage("", "Error running task: " + taskName);+ console.addException(taskName, e);+ }+ }+ finally {+ cleanupDecompiler();+ }+ }++ protected void comparePinnings(TaskMonitor monitor) throws CancelledException {+ monitor.setMessage("Comparing pinnings...");+ monitor.setIndeterminate(false);+ monitor.initialize(functionMap.size());++ for (Function left : functionMap.keySet()) {+ monitor.checkCancelled();++ Set<Function> tgtSet = functionMap.get(left);+ for (Function right : tgtSet) {+ monitor.checkCancelled();++ DecompileResults currentResults =+ getDecompilerResults(currentInterface, left, monitor);+ HighFunction currentHFunc = currentResults.getHighFunction();+ if (currentHFunc == null) {+ continue;+ }++ DecompileResults targetResults =+ getDecompilerResults(targetInterface, right, monitor);+ HighFunction targetHFunc = targetResults.getHighFunction();+ if (targetHFunc == null) {+ continue;+ }++ Pinning pin =+ Pinning.makePinning(currentHFunc, targetHFunc, false, false, true,+ TaskMonitor.DUMMY);++ for (String fieldName : offsets.keySet()) {+ Long offset = offsets.get(fieldName);+ AddressSet addresses = addressesByField.get(fieldName);+ results.putIfAbsent(offset, new HashMap<>());+ analyzePinning(pin, offset, addresses);+ }+ }+ monitor.increment();+ }+ //printResults();+ buildStructure();++ }++ private void initDecompiler() {+ DecompileOptions options = new DecompileOptions();++ currentInterface = new DecompInterface();+ currentInterface.setOptions(options);+ currentInterface.openProgram(currentProgram);++ targetInterface = new DecompInterface();+ targetInterface.setOptions(options);+ targetInterface.openProgram(targetProgram);+ }++ private void cleanupDecompiler() {+ if (currentInterface != null) {+ currentInterface.dispose();+ }+ if (targetInterface != null) {+ targetInterface.dispose();+ }+ decompilerResults.clear();+ }++ public DecompileResults getDecompilerResults(DecompInterface ifc, Function func,+ TaskMonitor monitor) {+ return decompilerResults+ .computeIfAbsent(ifc.getProgram(), _ -> new HashMap<>())+ .computeIfAbsent(func, f -> ifc.decompileFunction(f, DECOMPILER_TIMEOUT, monitor));+ }++ private void analyzePinning(Pinning pin, long offset, AddressSet addresses) {+ Map<DataVertex, DataVertex> originalPinMap = pin.getPinMap();+ Map<PcodeOp, DataVertex> op2vertex = new HashMap<>();+ Map<PcodeOp, DataVertex> opToVertexMap = new HashMap<>();++ for (DataVertex lv : originalPinMap.keySet()) {+ PcodeOpAST op = lv.getOp();+ if (op != null && !opToVertexMap.containsKey(op)) {+ opToVertexMap.put(op, originalPinMap.get(lv));+ op2vertex.put(op, lv);+ }+ }++ for (PcodeOp lop : opToVertexMap.keySet()) {+ DataVertex rv = opToVertexMap.get(lop);+ DataVertex lv = op2vertex.get(lop);+ if (addresses != null && addresses.contains(lop.getSeqnum().getTarget())) {+ compareConstants(lv, rv, offset);+ }+ }+ }++ private void compareConstants(DataVertex vertL, DataVertex vertR, long offset) {+ Set<Varnode> constantsL = new LinkedHashSet<>();+ collectConstantsByOperand(vertL.getOp(), new HashSet<>(), constantsL,+ vertL.getDepthPaired());++ Set<Varnode> constantsR = new LinkedHashSet<>();+ collectConstantsByOperand(vertR.getOp(), new HashSet<>(), constantsR,+ vertR.getDepthPaired());++ Set<Long> uniqL = extractConstants(constantsL);+ Set<Long> uniqR = extractConstants(constantsR);+ Set<Long> init = new TreeSet<>();+ init.addAll(uniqL);++ uniqR.removeAll(plugin.getExcludedOffsets());++ Set<Long> common = new HashSet<>(uniqL);+ common.retainAll(uniqR);++ // Remove what's in common+ uniqL.removeAll(common);+ uniqR.removeAll(common);++ if (uniqL.isEmpty() || uniqR.isEmpty()) {+ if (init.contains(offset)) {+ upcount(offset, offset);+ }+ return;+ }++ // Use obvious matches+ if (uniqL.size() == 1 && uniqL.contains(offset) && uniqR.size() == 1) {+ upcount(offset, uniqR.iterator().next());+ }+ }++ private void upcount(Long offsetL, Long offsetR) {+ results.computeIfAbsent(offsetL, _ -> new HashMap<>())+ .merge(offsetR, 1.0f, Float::sum);+ }++ private Set<Long> extractConstants(Set<Varnode> constants) {+ Set<Long> uniq = new TreeSet<>();+ for (Varnode vn : constants) {+ PcodeOp descendent = vn.getLoneDescend();+ if (descendent != null) {+ int opcode = descendent.getOpcode();+ if (opcode == PcodeOp.INDIRECT) {+ continue;+ }+ if (opcode == PcodeOp.PTRADD) {+ Varnode input1 = descendent.getInput(1);+ Varnode input2 = descendent.getInput(2);+ if (input1.isConstant() && input2.isConstant()) {+ uniq.add(Math.abs(input1.getOffset() * input2.getOffset()));+ continue;+ }+ }+ }+ long vnOffset = vn.getOffset();+ if (vnOffset == 0 || vnOffset == 1) {+ continue;+ }+ vnOffset = Math.abs(vnOffset);+ if (vnOffset > maxOffset) {+ continue;+ }+ uniq.add(vnOffset);+ }+ return uniq;+ }++ private void collectConstantsByOperand(PcodeOp op, Set<PcodeOp> visitedOps,+ Set<Varnode> constants, int depth) {+ if (depth < 0 || !visitedOps.add(op)) {+ return;+ }++ int nextDepth = depth - 1;+ int numInputs = op.getNumInputs();+ for (int i = 0; i < numInputs; i++) {+ Varnode input = op.getInput(i);+ if (input != null) {+ collectConstantsByVarnode(input, visitedOps, constants, nextDepth);+ }+ }+ }++ private void collectConstantsByVarnode(Varnode vn, Set<PcodeOp> visitedOps,+ Set<Varnode> constants, int depth) {+ if (depth < 0) {+ return;+ }+ if (vn.isConstant()) {+ constants.add(vn);+ }+ PcodeOp def = vn.getDef();+ if (def != null) {+ collectConstantsByOperand(def, visitedOps, constants, depth);+ }+ }++ private void buildStructure() {+ Structure original = plugin.getTargetDataType();+ if (original == null) {+ console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT,+ "Error: Target data type is missing.");+ return;+ }++ List<PinningMatch> matchHistory = new ArrayList<>();+ long max = -1L;+ for (Entry<Long, Map<Long, Float>> entry : results.entrySet()) {+ Long currentOffset = entry.getKey();+ DataTypeComponent c = original.getComponentAt(currentOffset.intValue());+ int srcOffset = entry.getKey().intValue();+ Map<Long, Float> targetMap = entry.getValue();+ for (Entry<Long, Float> targetEntry : targetMap.entrySet()) {+ Long targetOffset = targetEntry.getKey();+ if (targetOffset > max) {+ max = targetOffset + c.getLength();+ }+ Float weight = targetEntry.getValue();+ if (weight != null) {+ matchHistory.add(new PinningMatch(weight, srcOffset, targetOffset.intValue()));+ }+ }+ }++ matchHistory.sort((a, b) -> Float.compare(b.weight, a.weight));++ try (Transaction _ = targetProgram.openTransaction("Build structure")) {+ ProgramBasedDataTypeManager dtm = targetProgram.getDataTypeManager();+ Structure structure =+ new StructureDataType("RECOVERED_" + original.getName(), (int) max, dtm);++ Set<Integer> placed = new HashSet<>();+ Set<Integer> occupiedBytes = new HashSet<>();+ for (PinningMatch match : matchHistory) {+ if (placed.contains(match.sourceOffset)) {+ continue;+ }++ DataTypeComponent origComp = original.getComponentAt(match.sourceOffset);+ DataTypeComponent targetComp = structure.getComponentAt(match.targetOffset);++ DataType dataType = origComp.getDataType();+ DataType targetDataType = dtm.getDataType(dataType.getDataTypePath());++ if (targetComp != null && targetComp.isUndefined()) {+ if (targetDataType == null) {+ // Try to resolve or replicate the type context locally if path missing+ targetDataType =+ dtm.resolve(dataType, DataTypeConflictHandler.DEFAULT_HANDLER);+ }+ if (targetDataType == null) {+ targetDataType = DataType.DEFAULT;+ }++ int componentLength = targetDataType.getLength();+ if (componentLength <= 0) {+ componentLength = origComp.getLength();+ }++ // Check for overlapping byte targets (possibly overkill,, but...)+ boolean conflict = false;+ for (int i = 0; i < componentLength; i++) {+ if (occupiedBytes.contains(match.targetOffset + i)) {+ conflict = true;+ break;+ }+ }++ if (conflict) {+ console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT,+ "Overlapping target bounds for " + match + " : " ++ origComp + " using Byte");+ targetDataType = ByteDataType.dataType;+ componentLength = targetDataType.getLength();+ }++ try {+ structure.replaceAtOffset(match.targetOffset, targetDataType,+ targetDataType.getLength(), origComp.getFieldName(),+ origComp.getComment());+ placed.add(match.sourceOffset);+ for (int i = 0; i < componentLength; i++) {+ occupiedBytes.add(match.targetOffset + i);+ }+ }+ catch (IllegalArgumentException iae) {+ console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT,+ iae.getMessage());+ }+ }+ }++ console.addMessage(StructureRecoveryPlugin.REGENERATE_STRUCT,+ "Placed: " + placed.size() + " in " + structure.getName());+ DataTypeComponent[] definedComponents = structure.getDefinedComponents();+ DataTypeComponent last = definedComponents[definedComponents.length - 1];+ structure.setLength(last.getEndOffset() + 1);+ dtm.addDataType(structure, DataTypeConflictHandler.DEFAULT_HANDLER);+ }+ }+}diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/MatchFunctionsTask.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/MatchFunctionsTask.javanew file mode 100644index 0000000000..eb71dab092--- /dev/null+++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/MatchFunctionsTask.java@@ -0,0 +1,320 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.features.bsim.gui.structs;++import java.util.*;++import org.jgrapht.Graph;+import org.jgrapht.alg.interfaces.MatchingAlgorithm.Matching;+import org.jgrapht.alg.matching.MaximumWeightBipartiteMatching;+import org.jgrapht.graph.DefaultWeightedEdge;+import org.jgrapht.graph.SimpleWeightedGraph;++import db.Transaction;+import generic.lsh.vector.*;+import ghidra.app.decompiler.DecompileException;+import ghidra.app.services.ConsoleService;+import ghidra.features.bsim.gui.structs.StructureRecoveryPlugin.FunctionMatchOption;+import ghidra.features.bsim.query.*;+import ghidra.features.bsim.query.client.Configuration;+import ghidra.features.bsim.query.description.FunctionDescription;+import ghidra.program.model.address.Address;+import ghidra.program.model.listing.*;+import ghidra.util.Msg;+import ghidra.util.exception.CancelledException;+import ghidra.util.task.Task;+import ghidra.util.task.TaskMonitor;++public class MatchFunctionsTask extends Task {++ //functions with self significance below this bound will be skipped+ private double SELF_SIGNIFICANCE_BOUND = 15.0;+ //bsim database template determining the signature settings+ private String TEMPLATE_NAME = "medium_nosize";+ //these are analogous to the bounds in a bsim query+ private double MATCH_SIMILARITY_LOWER_BOUND = 0.2;+ //decrease this if you only want to see matches that aren't exact+ //for instance, when looking for changes between two versions of a program+ private double MATCH_SIMILARITY_UPPER_BOUND = 1.0;+ private double MATCH_CONFIDENCE_LOWER_BOUND = 0.0;++ // We'll consider solo matches or matches above this confidence+ private double MIN_CONFIDENCE = 20.0;++ private StructureRecoveryPlugin plugin;+ private ConsoleService console;++ private Program currentProgram;+ private Program targetProgram;+ private BookmarkManager srcBookmarks;+ private BookmarkManager tgtBookmarks;++ private Set<Function> functionsToMatch = new HashSet<>();+ Map<Function, Set<Function>> functionMap = new HashMap<>();++ public final static String TAG = "Function Match";++ public MatchFunctionsTask(StructureRecoveryPlugin plugin, Program target) {+ super("Match Functions", true, false, false);+ this.plugin = plugin;+ this.targetProgram = target;+ this.functionsToMatch = plugin.getFunctionsToMatch();+ this.console = plugin.getConsole();+ this.currentProgram = plugin.getCurrentProgram();+ this.SELF_SIGNIFICANCE_BOUND = plugin.getSelfSiginificanceBound();+ this.MATCH_SIMILARITY_LOWER_BOUND = plugin.getMatchSimilarityLowerBound();+ this.MATCH_SIMILARITY_UPPER_BOUND = plugin.getMatchSimilarityUpperBound();+ this.MATCH_CONFIDENCE_LOWER_BOUND = plugin.getMatchConfidenceLowerBound();+ this.MIN_CONFIDENCE = plugin.getMinConfidence();+ this.srcBookmarks = currentProgram.getBookmarkManager();+ this.tgtBookmarks = targetProgram.getBookmarkManager();+ }++ @Override+ public void run(TaskMonitor monitor) {+ String taskName = getTaskTitle();+ try {+ Thread.currentThread().setName(taskName);+ console.addMessage(taskName, "Running...");+ matchFunctions(monitor);+ console.addMessage(taskName, "Finished!");+ }+ catch (CancelledException e) {+ console.addMessage(taskName, "Cancelled by user.");+ }+ catch (Exception e) {+ if (!monitor.isCancelled()) {+ Msg.showError(this, null, getTaskTitle(), "Error running task: " + taskName ++ "\n" + e.getClass().getName() + ": " + e.getMessage(), e);+ console.addErrorMessage("", "Error running task: " + taskName);+ console.addException(taskName, e);+ }+ }+ }++ private void matchFunctions(TaskMonitor monitor)+ throws CancelledException, LSHException, DecompileException {+ Iterator<Function> functionIter = targetProgram.getFunctionManager().getFunctions(true);+ GenSignatures targetSigs =+ generateSignatures(targetProgram, functionIter, getVectorFactory(), monitor);++ List<LocalBSimMatch> matchResults =+ getMatchesTwoPrograms(functionsToMatch, targetSigs, monitor);++ FunctionMatchOption matchType = plugin.getOptionSoloMatchesOnly();+ if (matchType != FunctionMatchOption.MAX_WEIGHT) {+ defineMapFromSoloMatches(matchResults, matchType, monitor);+ }+ else {+ defineMap(matchResults, monitor);+ }+ }++ private void defineMapFromSoloMatches(List<LocalBSimMatch> matchResults,+ FunctionMatchOption matchType, TaskMonitor monitor) {+ boolean addBookmarks = plugin.getOptionFnMatchBookmarks();+ try (Transaction _ = currentProgram.openTransaction("add");+ Transaction _ = targetProgram.openTransaction("add")) {+ Set<String> processed = new HashSet<>();+ for (LocalBSimMatch match : matchResults) {+ Function left = match.sourceFunc();+ Set<Function> set = functionMap.computeIfAbsent(left, _ -> new HashSet<>());+ Function right = match.targetFunc();+ if (right != null && !matchType.equals(FunctionMatchOption.MAX_WEIGHT)) {+ set.add(right);+ Double sig = match.significance();+ if (addBookmarks && sig >= MIN_CONFIDENCE) {+ addBookmarks(left, right, processed);+ }+ }+ }+ for (Function left : functionMap.keySet()) {+ Set<Function> set = functionMap.get(left);+ if (set.size() == 1) {+ Function right = set.iterator().next();+ if (addBookmarks) {+ addBookmarks(left, right, processed);+ }+ }+ }+ }++ plugin.setFunctionMap(functionMap);+ }++ private void defineMap(List<LocalBSimMatch> matchResults, TaskMonitor monitor)+ throws CancelledException {+ Set<String> setLeft = new HashSet<>();+ Set<String> setRight = new HashSet<>();+ Graph<String, DefaultWeightedEdge> graph =+ new SimpleWeightedGraph<>(DefaultWeightedEdge.class);++ monitor.setMessage("Computing local BSim results...");+ monitor.setMaximum(matchResults.size());+ for (LocalBSimMatch match : matchResults) {+ monitor.checkCancelled();+ Function left = match.sourceFunc();+ Function right = match.targetFunc();+ if (right != null) {+ Double sig = match.significance();+ String leftID = "L" + Long.toString(left.getID());+ String rightID = "R" + Long.toString(right.getID());+ setLeft.add(leftID);+ graph.addVertex(leftID);+ setRight.add(rightID);+ graph.addVertex(rightID);+ addWeightedEdge(graph, leftID, rightID, sig);+ }+ }++ MaximumWeightBipartiteMatching<String, DefaultWeightedEdge> alg =+ new MaximumWeightBipartiteMatching<>(graph, setLeft, setRight);++ boolean addBookmarks = plugin.getOptionFnMatchBookmarks();+ try (Transaction _ = currentProgram.openTransaction("add");+ Transaction _ = targetProgram.openTransaction("add")) {++ FunctionManager sourceManager = currentProgram.getFunctionManager();+ FunctionManager targetManager = targetProgram.getFunctionManager();+ monitor.setMessage("Computing match...");+ Matching<String, DefaultWeightedEdge> matching = alg.getMatching();++ Set<String> processedTags = new HashSet<>();+ monitor.setMessage("Computing map...");+ monitor.setMaximum(matching.getEdges().size());+ for (DefaultWeightedEdge edge : matching.getEdges()) {+ monitor.checkCancelled();+ String edgeSource = graph.getEdgeSource(edge).substring(1);+ Function left = sourceManager.getFunction(Long.parseLong(edgeSource));+ String edgeTarget = graph.getEdgeTarget(edge).substring(1);+ Function right = targetManager.getFunction(Long.parseLong(edgeTarget));+ Set<Function> set = functionMap.computeIfAbsent(left, _ -> new HashSet<>());+ set.add(right);+ if (addBookmarks) {+ addBookmarks(left, right, processedTags);+ }+ }+ }++ plugin.setFunctionMap(functionMap);+ }++ private static void addWeightedEdge(Graph<String, DefaultWeightedEdge> graph, String source,+ String target, double weight) {+ DefaultWeightedEdge edge = graph.addEdge(source, target);+ if (edge != null) {+ graph.setEdgeWeight(edge, weight);+ }+ }++ private void addBookmarks(Function left, Function right, Set<String> processed) {+ String key = left.getID() + "->" + right.getID();+ if (processed.add(key)) {+ srcBookmarks.setBookmark(left.getEntryPoint(), TAG,+ Long.toString(left.getID()), Long.toString(right.getID()));+ tgtBookmarks.setBookmark(right.getEntryPoint(), TAG,+ Long.toString(left.getID()), Long.toString(right.getID()));+ }+ }++ public record LocalBSimMatch(Function sourceFunc, Function targetFunc, double sim,+ double significance) {}++ private List<LocalBSimMatch> getMatchesTwoPrograms(Set<Function> srcFuncs,+ GenSignatures targetSigs, TaskMonitor monitor)+ throws LSHException, DecompileException, CancelledException {+ LSHVectorFactory vectorFactory = getVectorFactory();+ GenSignatures srcSigs =+ generateSignatures(currentProgram, srcFuncs.iterator(), vectorFactory, monitor);+ Iterator<FunctionDescription> sourceDescripts =+ srcSigs.getDescriptionManager().listAllFunctions();++ VectorCompare vecCompare = new VectorCompare();+ List<LocalBSimMatch> bsimMatches = new ArrayList<>();++ List<FunctionDescription> targetList = new ArrayList<>();+ Iterator<FunctionDescription> targetDescriptsIter =+ targetSigs.getDescriptionManager().listAllFunctions();+ while (targetDescriptsIter.hasNext()) {+ monitor.checkCancelled();+ targetList.add(targetDescriptsIter.next());+ }++ while (sourceDescripts.hasNext()) {+ monitor.checkCancelled();+ FunctionDescription srcDesc = sourceDescripts.next();++ //skip if self-significance too small+ LSHVector srcVector = srcDesc.getSignatureRecord().getLSHVector();+ if (vectorFactory.getSelfSignificance(srcVector) <= SELF_SIGNIFICANCE_BOUND) {+ continue;+ }++ Function srcFunc = getFunction(currentProgram, srcDesc.getAddress());+ for (FunctionDescription targetDesc : targetList) {+ //skip if self-significance too small+ LSHVector targetVector = targetDesc.getSignatureRecord().getLSHVector();+ if (vectorFactory.getSelfSignificance(targetVector) <= SELF_SIGNIFICANCE_BOUND) {+ continue;+ }+ double sim = srcVector.compare(targetVector, vecCompare);+ double sig = vectorFactory.calculateSignificance(vecCompare);+ if (sig >= MATCH_CONFIDENCE_LOWER_BOUND && MATCH_SIMILARITY_LOWER_BOUND <= sim &&+ sim <= MATCH_SIMILARITY_UPPER_BOUND) {+ Function targetFunc = getFunction(targetProgram, targetDesc.getAddress());+ if (targetFunc != null) {+ bsimMatches.add(new LocalBSimMatch(srcFunc, targetFunc, sim, sig));+ }+ else {+ Msg.warn(this, "Function not found for " + targetDesc.getAddress());+ }+ }+ monitor.checkCancelled();+ }+ }+ return bsimMatches;+ }++ private Function getFunction(Program program, long offset) {+ Address addr = program.getAddressFactory().getDefaultAddressSpace().getAddress(offset);+ return program.getFunctionManager().getFunctionAt(addr);+ }++ private LSHVectorFactory getVectorFactory() throws LSHException {+ LSHVectorFactory vectorFactory = FunctionDatabase.generateLSHVectorFactory();+ Configuration config = FunctionDatabase.loadConfigurationTemplate(TEMPLATE_NAME);+ vectorFactory.set(config.weightfactory, config.idflookup, config.info.settings);+ return vectorFactory;+ }++ private GenSignatures generateSignatures(Program program, Iterator<Function> funcs,+ LSHVectorFactory vectorFactory, TaskMonitor monitor)+ throws LSHException, DecompileException {+ GenSignatures gensig = null;+ try {+ int count = program.getFunctionManager().getFunctionCount();+ gensig = new GenSignatures(false);+ gensig.setVectorFactory(vectorFactory);+ gensig.openProgram(program, null, null, null, null, null);+ gensig.scanFunctions(funcs, count, monitor);+ return gensig;+ }+ finally {+ gensig.dispose();+ }+ }++}diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/RetrieveUsesTask.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/RetrieveUsesTask.javanew file mode 100644index 0000000000..d532d44c8c--- /dev/null+++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/RetrieveUsesTask.java@@ -0,0 +1,115 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.features.bsim.gui.structs;++import java.util.*;++import ghidra.app.plugin.core.navigation.locationreferences.LocationReference;+import ghidra.app.plugin.core.navigation.locationreferences.ReferenceUtils;+import ghidra.app.services.ConsoleService;+import ghidra.app.services.FieldMatcher;+import ghidra.program.model.address.Address;+import ghidra.program.model.address.AddressSet;+import ghidra.program.model.data.Structure;+import ghidra.program.model.listing.*;+import ghidra.util.Msg;+import ghidra.util.datastruct.SetAccumulator;+import ghidra.util.exception.CancelledException;+import ghidra.util.task.Task;+import ghidra.util.task.TaskMonitor;++public class RetrieveUsesTask extends Task {++ private StructureRecoveryPlugin plugin;+ private Structure targetDataType;+ private ConsoleService console;+ private Program currentProgram;++ public final static String TAG = "Structure Use";++ public RetrieveUsesTask(StructureRecoveryPlugin plugin) {+ super("Retrieve Structure Uses", true, false, false);+ this.plugin = plugin;+ this.targetDataType = plugin.getTargetDataType();+ this.console = plugin.getConsole();+ this.currentProgram = plugin.getCurrentProgram();+ }++ @Override+ public void run(TaskMonitor monitor) {+ String taskName = getTaskTitle();+ Thread.currentThread().setName(taskName);+ try {+ console.addMessage(taskName, "Running...");+ retrieveReferences(monitor);+ console.addMessage(taskName, "Finished!");+ }+ catch (CancelledException e) {+ console.addMessage(taskName, "Cancelled by user.");+ }+ catch (Exception e) {+ if (!monitor.isCancelled()) {+ Msg.showError(this, null, getTaskTitle(), "Error running task: " + taskName ++ "\n" + e.getClass().getName() + ": " + e.getMessage(), e);+ console.addErrorMessage("", "Error running task: " + taskName);+ console.addException(taskName, e);+ }+ }+ }++ protected void retrieveReferences(TaskMonitor monitor) throws CancelledException {+ FieldMatcher fieldMatcher = new FieldMatcher(targetDataType);++ SetAccumulator<LocationReference> accumulator = new SetAccumulator<>();+ ReferenceUtils.findDataTypeFieldReferences(accumulator, fieldMatcher, currentProgram, true,+ monitor);++ Set<Function> toMatch = new HashSet<>();+ Map<String, Set<Function>> byField = new HashMap<>();+ Map<String, AddressSet> toAddressSet = new HashMap<>();++ boolean useBookmarks = plugin.getOptionStructUseBookmarks();+ currentProgram.withTransaction("Add Structure Use Bookmarks", () -> {+ BookmarkManager bookmarkManager = currentProgram.getBookmarkManager();+ FunctionManager functionManager = currentProgram.getFunctionManager();+ String typeName = targetDataType.getName();++ for (LocationReference lr : accumulator.get()) {+ String fieldName = lr.getFieldName();+ if (fieldName == null) {+ continue;+ }+ Address locationOfUse = lr.getLocationOfUse();+ Function f = functionManager.getFunctionContaining(locationOfUse);+ if (f == null) {+ continue;+ }+ toAddressSet.computeIfAbsent(fieldName, _ -> new AddressSet())+ .add(locationOfUse);+ byField.computeIfAbsent(fieldName, _ -> new HashSet<Function>()).add(f);+ toMatch.add(f);+ if (useBookmarks) {+ bookmarkManager.setBookmark(locationOfUse, TAG,+ "USES_" + typeName, fieldName);+ }+ }+ });++ plugin.setFunctionsToMatch(toMatch);+ plugin.setFunctionsByField(byField);+ plugin.setAddressesByField(toAddressSet);+ }+}diff --git a/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/StructureRecoveryPlugin.java b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/StructureRecoveryPlugin.javanew file mode 100644index 0000000000..f2c452fd42--- /dev/null+++ b/Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/structs/StructureRecoveryPlugin.java@@ -0,0 +1,686 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.features.bsim.gui.structs;++import static ghidra.framework.main.DataTreeDialogType.*;++import java.io.IOException;+import java.util.*;+import java.util.Map.Entry;++import db.Transaction;+import docking.DockingWindowManager;+import docking.action.DockingAction;+import docking.action.MenuData;+import docking.action.builder.ActionBuilder;+import ghidra.app.CorePluginPackage;+import ghidra.app.events.*;+import ghidra.app.merge.structures.StructureMergeDialog;+import ghidra.app.plugin.PluginCategoryNames;+import ghidra.app.plugin.ProgramPlugin;+import ghidra.app.services.ConsoleService;+import ghidra.app.util.datatype.DataTypeSelectionDialog;+import ghidra.framework.main.DataTreeDialog;+import ghidra.framework.model.DomainFile;+import ghidra.framework.model.DomainFileFilter;+import ghidra.framework.options.*;+import ghidra.framework.plugintool.*;+import ghidra.framework.plugintool.util.PluginStatus;+import ghidra.program.model.address.*;+import ghidra.program.model.data.*;+import ghidra.program.model.listing.*;+import ghidra.program.util.ProgramLocation;+import ghidra.util.*;+import ghidra.util.bean.opteditor.OptionsVetoException;+import ghidra.util.data.DataTypeParser.AllowedDataTypes;+import ghidra.util.exception.CancelledException;+import ghidra.util.exception.VersionException;+import ghidra.util.task.TaskMonitor;++/**+ * Plugin for recovering structure layouts across programs.+ */+//@formatter:off+@PluginInfo(+ status = PluginStatus.UNSTABLE,+ packageName = CorePluginPackage.NAME,+ category = PluginCategoryNames.ANALYSIS,+ shortDescription = "StructureRecovery",+ description = "Plugin for recovering structure layouts across programs",+ servicesRequired = {+ ConsoleService.class+ },+ eventsConsumed = {+ ProgramActivatedPluginEvent.class, ProgramOpenedPluginEvent.class,+ ProgramLocationPluginEvent.class, ProgramSelectionPluginEvent.class,+ ProgramClosedPluginEvent.class+ })+//@formatter:on++public class StructureRecoveryPlugin extends ProgramPlugin implements OptionsChangeListener {++ static final String TAG_STRUCTURE_USE = "Tag structure use";+ static final String MATCH_FUNCTIONS = "Match functions";+ static final String REGENERATE_STRUCT = "Generate structure";+ static final String COMPARE_STRUCTURES = "Compare new structure";++ private static final String FUNCTIONS_MAP_BKMKS = "Function matches: map to bookmarks";+ private static final String FUNCTIONS_BKMKS_MAP = "Function matches: bookmarks to map";+ private static final String FUNCTIONS_BKMKS_DEL = "Function matches: remove bookmarks";+ private static final String USES_MAP_BKMKS = "Structure uses: map to bookmarks";+ private static final String USES_BKMKS_MAP = "Structure uses: bookmarks to map";+ private static final String USES_BKMKS_DEL = "Structure uses: remove bookmarks";++ public final static String HELP_LOCATION = "StructureRecovery";++ private static final String DEFAULT_STRUCTURE = "Structure to recover";+ private static final String ADD_STRUCTURE_USE_BKMKS = "Add structure-use bookmarks";+ private static final String ADD_FUNCTION_MATCH_BKMKS = "Add function-match bookmarks";+ private static final String SELF_SIGNIFICANCE_BOUND = "Self-significance bound";+ private static final String MATCH_SIMILARITY_LOWER_BOUND = "Match-similarity lower bound";+ private static final String MATCH_SIMILARITY_UPPER_BOUND = "Match-similarity upper bound";+ private static final String MATCH_CONFIDENCE_LOWER_BOUND = "Match-confidence lower bound";+ private static final String MIN_CONFIDENCE = "Min confidence";+ private static final String MAX_OFFSET = "Max structure offset";+ private static final String EXCLUDED_OFFSETS = "Excluded offsets (CSV hex values)";+ private static final String MATCH_ON = "Match on";++ public enum FunctionMatchOption {+ SOLO_MATCHES, SOLO_OR_MIN_CONFIDENCE, MAX_WEIGHT+ }++ Program targetProgram;++ Map<Function, Set<Function>> functionMap = new HashMap<>();+ Map<Function, Float> confidence = new HashMap<>();++ private Function currentFunction;+ private ConsoleService consoleService;++ private Structure targetDataType;+ private Map<String, Long> offsets = new HashMap<>();+ private Map<Long, String> names = new HashMap<>();+ private Set<Function> srcFunctionsToMatch = new HashSet<>();+ private Map<String, Set<Function>> srcFunctionsByField = new HashMap<>();+ private Map<String, AddressSet> srcAddressesByField = new HashMap<>();+ private ToolOptions options;+ private DockingAction tagAction;+ private DockingAction regenerateAction;+ private DockingAction compareAction;++ public StructureRecoveryPlugin(PluginTool tool) {+ super(tool);+ createActions();+ getOptions();+ }++ public Function getCurrentFunction() {+ return currentFunction;+ }++ @Override+ protected void programActivated(Program program) {+ currentProgram = program;+ }++ @Override+ protected void programClosed(Program program) {+ currentProgram = null;+ }++ private void createActions() {++ tagAction = new ActionBuilder(TAG_STRUCTURE_USE, HELP_LOCATION)+ .enabledWhen(_ -> getCurrentProgram() != null)+ .onAction(_ -> {+ if (checkDataType()) {+ tool.execute(new RetrieveUsesTask(StructureRecoveryPlugin.this));+ }+ })+ .menuGroup("A", "a")+ .menuPath("Tools", "Structure recovery", TAG_STRUCTURE_USE)+ .buildAndInstall(tool);++ new ActionBuilder(MATCH_FUNCTIONS, HELP_LOCATION)+ .enabledWhen(_ -> getCurrentProgram() != null && !srcFunctionsToMatch.isEmpty())+ .onAction(_ -> {+ tool.execute(+ new MatchFunctionsTask(StructureRecoveryPlugin.this, getTargetProgram()));+ })+ .menuGroup("A", "b")+ .menuPath("Tools", "Structure recovery", MATCH_FUNCTIONS)+ .buildAndInstall(tool);++ regenerateAction = new ActionBuilder(REGENERATE_STRUCT, HELP_LOCATION)+ .enabledWhen(_ -> getCurrentProgram() != null && !functionMap.isEmpty() &&+ targetProgram != currentProgram)+ .onAction(_ -> {+ if (checkDataType()) {+ tool.execute(new ComparePinningsTask(StructureRecoveryPlugin.this));+ }+ })+ .menuGroup("A", "c")+ .menuPath("Tools", "Structure recovery", REGENERATE_STRUCT)+ .buildAndInstall(tool);++ compareAction = new ActionBuilder(COMPARE_STRUCTURES, HELP_LOCATION)+ .enabledWhen(_ -> getCurrentProgram() != null)+ .onAction(_ -> {+ compareStructures();+ })+ .menuGroup("A", "d")+ .menuPath("Tools", "Structure recovery", COMPARE_STRUCTURES)+ .buildAndInstall(tool);++ new ActionBuilder(USES_BKMKS_MAP, HELP_LOCATION)+ .enabledWhen(_ -> getCurrentProgram() != null)+ .onAction(_ -> {+ if (checkDataType()) {+ getConsole().addMessage(USES_BKMKS_MAP, "Running...");+ regenerateUsesMapFromBookmarks();+ getConsole().addMessage(USES_BKMKS_MAP, "Finished!");+ }+ })+ .menuGroup("B", "a")+ .menuPath("Tools", "Structure recovery", USES_BKMKS_MAP)+ .buildAndInstall(tool);++ new ActionBuilder(USES_MAP_BKMKS, HELP_LOCATION)+ .enabledWhen(_ -> getCurrentProgram() != null && !srcFunctionsByField.isEmpty())+ .onAction(_ -> {+ getConsole().addMessage(USES_MAP_BKMKS, "Running...");+ generateUsesMapBookmarks();+ getConsole().addMessage(USES_MAP_BKMKS, "Finished!");+ })+ .menuGroup("B", "b")+ .menuPath("Tools", "Structure recovery", USES_MAP_BKMKS)+ .buildAndInstall(tool);++ new ActionBuilder(FUNCTIONS_BKMKS_MAP, HELP_LOCATION)+ .enabledWhen(_ -> getCurrentProgram() != null)+ .onAction(_ -> {+ getConsole().addMessage(FUNCTIONS_BKMKS_MAP, "Running...");+ regenerateFunctionMapFromBookmarks();+ getConsole().addMessage(FUNCTIONS_BKMKS_MAP, "Finished!");+ })+ .menuGroup("B", "c")+ .menuPath("Tools", "Structure recovery", FUNCTIONS_BKMKS_MAP)+ .buildAndInstall(tool);++ new ActionBuilder(FUNCTIONS_MAP_BKMKS, HELP_LOCATION)+ .enabledWhen(_ -> getCurrentProgram() != null && !functionMap.isEmpty())+ .onAction(_ -> {+ getConsole().addMessage(FUNCTIONS_MAP_BKMKS, "Running...");+ generateFunctionMapBookmarks();+ getConsole().addMessage(FUNCTIONS_MAP_BKMKS, "Finished!");+ })+ .menuGroup("B", "d")+ .menuPath("Tools", "Structure recovery", FUNCTIONS_MAP_BKMKS)+ .buildAndInstall(tool);++ new ActionBuilder(USES_BKMKS_DEL, HELP_LOCATION)+ .enabledWhen(_ -> getCurrentProgram() != null)+ .onAction(_ -> {+ wipeUsesMapBookmarks();+ })+ .menuGroup("B", "e")+ .menuPath("Tools", "Structure recovery", USES_BKMKS_DEL)+ .buildAndInstall(tool);++ new ActionBuilder(FUNCTIONS_BKMKS_DEL, HELP_LOCATION)+ .enabledWhen(_ -> getCurrentProgram() != null)+ .onAction(_ -> {+ wipeFunctionMapBookmarks();+ })+ .menuGroup("B", "f")+ .menuPath("Tools", "Structure recovery", FUNCTIONS_BKMKS_DEL)+ .buildAndInstall(tool);++ }++ @Override+ public void optionsChanged(ToolOptions options, String optionName, Object oldValue,+ Object newValue) throws OptionsVetoException {+ if (optionName.equals(DEFAULT_STRUCTURE)) {+ checkDataType();+ boolean valid = targetDataType != null;+ String tagCmd = !valid ? TAG_STRUCTURE_USE : "Tag " + targetDataType.getName() + " use";+ String regenCmd = !valid ? REGENERATE_STRUCT+ : "Generate " + targetDataType.getName();+ if (targetProgram != null) {+ regenCmd += " (" + targetProgram + ")";+ }+ String compCmd = !valid ? COMPARE_STRUCTURES+ : "Compare " + targetDataType.getName() + " structures";+ tagAction.setMenuBarData(+ new MenuData(new String[] { "Tools", "Structure recovery", tagCmd }));+ regenerateAction.setMenuBarData(new MenuData(+ new String[] { "Tools", "Structure recovery", regenCmd }));+ compareAction.setMenuBarData(new MenuData(+ new String[] { "Tools", "Structure recovery", compCmd }));+ }+ }++ @Override+ public Program getCurrentProgram() {+ return currentProgram;+ }++ @Override+ public void close() {+ if (targetProgram != null) {+ targetProgram.release(this);+ }+ }++ @Override+ public void processEvent(PluginEvent event) {+ super.processEvent(event);++ //Msg.info(this, "TaintPlugin -> processEvent: " + event.toString() );++ if (event instanceof ProgramClosedPluginEvent) {+ if (currentProgram != null) {+ currentProgram.release(this);+ }+ return;+ }++ if (event instanceof ProgramActivatedPluginEvent) {+ currentProgram = ((ProgramActivatedPluginEvent) event).getActiveProgram();+ }+ else if (event instanceof ProgramLocationPluginEvent) {+ ProgramLocation location = ((ProgramLocationPluginEvent) event).getLocation();+ Address address = location.getAddress();++ if (address.isExternalAddress()) {+ // ignore external functions when it comes to taint.+ return;+ }++ if (currentProgram != null) {+ // The user loaded a program for analysis.+ Listing listing = currentProgram.getListing();+ Function f = listing.getFunctionContaining(address);+ // We are in function f+ if (currentFunction == null || !currentFunction.equals(f)) {+ currentFunction = f;+ }+ }+ }+ }++ private boolean checkDataType() {+ if (targetDataType != null) {+ return true;+ }+ if (currentProgram == null) {+ Msg.showWarn(this, null, "No Program",+ "Please open a program before selecting a structure.");+ return false;+ }+ DataTypeManager dtm = currentProgram.getDataTypeManager();+ String structureName = getDefaultStructureName();+ DataType dataType = null;+ if (!structureName.equals("")) {+ dataType = dtm.getDataType(structureName);+ if (dataType == null) {+ List<DataType> list = new ArrayList<>();+ dtm.findDataTypes(structureName, list);+ if (list.isEmpty()) {+ Msg.error(this, dataType + " in ToolOptions is not valid.");+ return false;+ }+ dataType = list.get(0);+ }+ }+ else {+ DataTypeSelectionDialog selectionDialog =+ new DataTypeSelectionDialog(tool, dtm, -1, AllowedDataTypes.FIXED_LENGTH);+ tool.showDialog(selectionDialog);+ dataType = selectionDialog.getUserChosenDataType();+ if (dataType == null) {+ return false;+ }+ options.setString(DEFAULT_STRUCTURE, dataType.getPathName());+ }+ if (dataType instanceof Structure struct) {+ targetDataType = struct;+ offsets.clear();+ names.clear();+ for (DataTypeComponent dtc : targetDataType.getComponents()) {+ if (dtc.getFieldName() != null && dtc.getOffset() > 0) {+ offsets.put(dtc.getFieldName(), (long) dtc.getOffset());+ names.put((long) dtc.getOffset(), dtc.getFieldName());+ }+ }+ return true;+ }+ return false;+ }++ public Program loadTargetProgram() throws VersionException, IOException, CancelledException {+ DomainFileFilter filter = df -> Program.class.isAssignableFrom(df.getDomainObjectClass());+ DataTreeDialog dtd = new DataTreeDialog(null, "Target Program", OPEN, filter);+ dtd.show();+ if (dtd.wasCancelled()) {+ return null;+ }++ DomainFile df = dtd.getDomainFile();+ return (Program) df.getDomainObject(this, true, false, TaskMonitor.DUMMY);+ }++ private void generateFunctionMapBookmarks() {+ BookmarkManager srcBookmarks = currentProgram.getBookmarkManager();+ BookmarkManager tgtBookmarks = targetProgram.getBookmarkManager();+ try (Transaction _ = currentProgram.openTransaction("add")) {+ try (Transaction _ = targetProgram.openTransaction("add")) {+ for (Entry<Function, Set<Function>> entry : functionMap.entrySet()) {+ Function left = entry.getKey();+ for (Function right : entry.getValue()) {+ srcBookmarks.setBookmark(left.getEntryPoint(), MatchFunctionsTask.TAG,+ Long.toString(left.getID()), Long.toString(right.getID()));+ tgtBookmarks.setBookmark(right.getEntryPoint(), MatchFunctionsTask.TAG,+ Long.toString(left.getID()), Long.toString(right.getID()));+ }+ }+ }+ }+ }++ private void regenerateFunctionMapFromBookmarks() {+ getTargetProgram();+ FunctionManager srcMgr = currentProgram.getFunctionManager();+ FunctionManager tgtMgr = targetProgram.getFunctionManager();+ boolean adds = false;+ BookmarkManager bkmkMgr = currentProgram.getBookmarkManager();+ Iterator<Bookmark> iterator = bkmkMgr.getBookmarksIterator(MatchFunctionsTask.TAG);+ while (iterator.hasNext()) {+ Bookmark mark = iterator.next();+ try {+ Function left = srcMgr.getFunction(Long.parseLong(mark.getCategory()));+ if (left == null) {+ continue;+ }+ Set<Function> set = functionMap.computeIfAbsent(left, _ -> new HashSet<>());+ String tgtId = mark.getComment();+ Function right = tgtMgr.getFunction(Long.parseLong(tgtId));+ if (right == null) {+ continue;+ }+ set.add(right);+ adds = true;+ }+ catch (NumberFormatException e) {+ // Skip+ }+ }+ if (!adds) {+ getConsole().addErrorMessage(FUNCTIONS_BKMKS_MAP,+ "Map empty - check function id bookmarks");+ }+ }++ private void wipeFunctionMapBookmarks() {+ try (Transaction _ = currentProgram.openTransaction("wipeCurrent")) {+ getTargetProgram();+ BookmarkManager srcBookmarks = currentProgram.getBookmarkManager();+ srcBookmarks.removeBookmarks(MatchFunctionsTask.TAG);+ }+ getTargetProgram();+ try (Transaction _ = targetProgram.openTransaction("wipeTarget")) {+ getTargetProgram();+ BookmarkManager tgtBookmarks = targetProgram.getBookmarkManager();+ tgtBookmarks.removeBookmarks(MatchFunctionsTask.TAG);+ }+ }++ private void generateUsesMapBookmarks() {+ BookmarkManager bookmarkManager = currentProgram.getBookmarkManager();+ try (Transaction _ = currentProgram.openTransaction("add")) {+ for (Entry<String, Set<Function>> entry : srcFunctionsByField.entrySet()) {+ String fname = entry.getKey();+ AddressSet addressSet = srcAddressesByField.get(fname);+ if (addressSet == null) {+ continue;+ }+ Iterator<Function> iterator = entry.getValue().iterator();+ while (iterator.hasNext()) {+ Function f = iterator.next();+ AddressSet intersect = addressSet.intersect(f.getBody());+ if (intersect == null) {+ continue;+ }+ // Add a bookmark for each intersecting address range's minimum address so+ // it can be parsed back into a single address when rebuilding the map.+ for (AddressRange range : intersect.getAddressRanges()) {+ bookmarkManager.setBookmark(range.getMinAddress(), RetrieveUsesTask.TAG,+ "USES_" + targetDataType.getName(), fname);+ }+ }+ }+ }+ }++ private void regenerateUsesMapFromBookmarks() {+ FunctionManager srcMgr = currentProgram.getFunctionManager();+ BookmarkManager bkmkMgr = currentProgram.getBookmarkManager();+ Iterator<Bookmark> iterator = bkmkMgr.getBookmarksIterator(RetrieveUsesTask.TAG);+ while (iterator.hasNext()) {+ Bookmark mark = iterator.next();+ AddressSet aset =+ srcAddressesByField.computeIfAbsent(mark.getComment(), _ -> new AddressSet());+ Address address = mark.getAddress();+ if (address != null) {+ aset.add(address);+ }+ Set<Function> set =+ srcFunctionsByField.computeIfAbsent(mark.getComment(), _ -> new HashSet<>());+ Function f = srcMgr.getFunctionContaining(address);+ if (f != null) {+ set.add(f);+ srcFunctionsToMatch.add(f);+ }+ }+ if (srcFunctionsToMatch.isEmpty()) {+ getConsole().addErrorMessage(USES_BKMKS_MAP, "Map empty - check USES bookmarks");+ }+ }++ private void wipeUsesMapBookmarks() {+ try (Transaction _ = currentProgram.openTransaction("wipe")) {+ BookmarkManager bkmkMgr = currentProgram.getBookmarkManager();+ bkmkMgr.removeBookmarks(RetrieveUsesTask.TAG);+ }+ }++ private void compareStructures() {+ checkDataType();+ getTargetProgram();+ if (targetProgram == null) {+ consoleService.addErrorMessage("Compare Structures", "Target program is null.");+ return;+ }+ DataTypeManager dtm = targetProgram.getDataTypeManager();+ String structureName = "/RECOVERED_" + targetDataType.getName();+ DataType dataType = dtm.getDataType(structureName);+ if (dataType == null) {+ consoleService.addErrorMessage("Compare Structures", structureName + " not found.");+ return;+ }+ if (dataType instanceof Structure recoveredStructure) {+ StructureMergeDialog dialog =+ new StructureMergeDialog("Compare Structures", recoveredStructure, targetDataType,+ _ -> {+ //IGNORE+ });+ DockingWindowManager.showDialog(dialog);+ }+ }++ public Map<String, Long> getOffsets() {+ return offsets;+ }++ public Map<Long, String> getFieldNames() {+ return names;+ }++ public Map<Function, Set<Function>> getFunctionMap() {+ return functionMap;+ }++ public void setFunctionMap(Map<Function, Set<Function>> map) {+ this.functionMap = map;+ }++ public Map<String, Set<Function>> getFunctionsByField() {+ return srcFunctionsByField;+ }++ public void setFunctionsByField(Map<String, Set<Function>> functionsByField) {+ this.srcFunctionsByField = functionsByField;+ }++ public Map<String, AddressSet> getAddressesByField() {+ return srcAddressesByField;+ }++ public void setAddressesByField(Map<String, AddressSet> toAddressSet) {+ this.srcAddressesByField = toAddressSet;+ }++ public Set<Function> getFunctionsToMatch() {+ return srcFunctionsToMatch;+ }++ public void setFunctionsToMatch(Set<Function> srcFunctionsToMatch) {+ this.srcFunctionsToMatch = srcFunctionsToMatch;+ }++ public Structure getTargetDataType() {+ return targetDataType;+ }++ public ConsoleService getConsole() {+ if (consoleService == null) {+ consoleService = tool.getService(ConsoleService.class);+ }+ return consoleService;+ }++ public Program getTargetProgram() {+ if (targetProgram == null) {+ Swing.runNow(() -> {+ try {+ targetProgram = loadTargetProgram();+ }+ catch (VersionException | CancelledException | IOException e) {+ getConsole().addMessage("Load target", e.getMessage());+ }+ });+ }+ return targetProgram;+ }++ private void getOptions() {+ options = tool.getOptions("Structure Recovery");+ HelpLocation help = new HelpLocation(getName(), "Options");+ options.registerOption(DEFAULT_STRUCTURE, OptionType.STRING_TYPE, "", help,+ DEFAULT_STRUCTURE);+ options.registerOption(ADD_STRUCTURE_USE_BKMKS, OptionType.BOOLEAN_TYPE, true, help,+ "Generate structure-use function bookmarks by default");+ options.registerOption(ADD_FUNCTION_MATCH_BKMKS, OptionType.BOOLEAN_TYPE, true, help,+ "Generate function-match function bookmarks by default");+ options.registerOption(SELF_SIGNIFICANCE_BOUND, OptionType.DOUBLE_TYPE, 15.0, help,+ "Functions with self significance below this bound will be skipped");+ options.registerOption(MATCH_SIMILARITY_LOWER_BOUND, OptionType.DOUBLE_TYPE, 0.3, help,+ "Lower threshold equates to looser matches");+ options.registerOption(MATCH_SIMILARITY_UPPER_BOUND, OptionType.DOUBLE_TYPE, 1.0, help,+ "Decrease this if you only want to see matches that aren't exact");+ options.registerOption(MATCH_CONFIDENCE_LOWER_BOUND, OptionType.DOUBLE_TYPE, 0.0, help,+ MATCH_CONFIDENCE_LOWER_BOUND);+ options.registerOption(MIN_CONFIDENCE, OptionType.DOUBLE_TYPE, 20.0, help,+ "Consider matches above this confidence");+ options.registerOption(MAX_OFFSET, OptionType.LONG_TYPE, 100000L, help,+ "Ignore constants above this value as probable addresses");+ options.registerOption(EXCLUDED_OFFSETS, OptionType.STRING_TYPE, "", help,+ "Useful for structures using boxed randomized fields");+ options.registerOption(MATCH_ON, OptionType.ENUM_TYPE,+ FunctionMatchOption.SOLO_OR_MIN_CONFIDENCE, help,+ "Match using solo matches, solo+confidence, bipartite pairing");+ options.addOptionsChangeListener(this);+ }++ public String getDefaultStructureName() {+ return options.getString(DEFAULT_STRUCTURE, "");+ }++ public boolean getOptionStructUseBookmarks() {+ return options.getBoolean(ADD_STRUCTURE_USE_BKMKS, true);+ }++ public boolean getOptionFnMatchBookmarks() {+ return options.getBoolean(ADD_FUNCTION_MATCH_BKMKS, true);+ }++ public double getSelfSiginificanceBound() {+ return options.getDouble(SELF_SIGNIFICANCE_BOUND, 15.0);+ }++ public double getMatchSimilarityLowerBound() {+ return options.getDouble(MATCH_SIMILARITY_LOWER_BOUND, 0.3);+ }++ public double getMatchSimilarityUpperBound() {+ return options.getDouble(MATCH_SIMILARITY_UPPER_BOUND, 1.0);+ }++ public double getMatchConfidenceLowerBound() {+ return options.getDouble(MATCH_CONFIDENCE_LOWER_BOUND, 0.0);+ }++ public double getMinConfidence() {+ return options.getDouble(MIN_CONFIDENCE, 20.0);+ }++ public long getMaxOffset() {+ return options.getLong(MAX_OFFSET, 100000);+ }++ public Set<Long> getExcludedOffsets() {+ String offstr = options.getString(EXCLUDED_OFFSETS, "");+ String[] split = offstr.split(",");+ Set<Long> excluded = new HashSet<>();+ for (String offset : split) {+ if (!offset.equals("")) {+ try {+ excluded.add(Long.parseLong(offset.trim(), 16));+ }+ catch (NumberFormatException e) {+ // Skip+ }+ }+ }+ return excluded;+ }++ public FunctionMatchOption getOptionSoloMatchesOnly() {+ return options.getEnum(MATCH_ON, FunctionMatchOption.SOLO_OR_MIN_CONFIDENCE);+ }++}diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/services/DataTypeReferenceFinder.java b/Ghidra/Features/Base/src/main/java/ghidra/app/services/DataTypeReferenceFinder.javaindex 1b02584827..c2d280754f 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/app/services/DataTypeReferenceFinder.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/services/DataTypeReferenceFinder.java@@ -4,9 +4,9 @@* Licensed under the Apache License, Version 2.0 (the "License");* you may not use this file except in compliance with the License.* You may obtain a copy of the License at- *+ ** http://www.apache.org/licenses/LICENSE-2.0- *+ ** Unless required by applicable law or agreed to in writing, software* distributed under the License is distributed on an "AS IS" BASIS,* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.@@ -84,4 +84,5 @@ public interface DataTypeReferenceFinder extends ExtensionPoint {*/public void findReferences(Program program, FieldMatcher fieldMatcher,Consumer<DataTypeReference> callback, TaskMonitor monitor) throws CancelledException;+}diff --git a/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/DataVertex.java b/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/DataVertex.javaindex c07918cf81..30960505fb 100755--- a/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/DataVertex.java+++ b/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/DataVertex.java@@ -4,9 +4,9 @@* Licensed under the Apache License, Version 2.0 (the "License");* you may not use this file except in compliance with the License.* You may obtain a copy of the License at- *+ ** http://www.apache.org/licenses/LICENSE-2.0- *+ ** Unless required by applicable law or agreed to in writing, software* distributed under the License is distributed on an "AS IS" BASIS,* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.@@ -212,4 +212,34 @@ public class DataVertex {}return PcodeOp.isCommutative(opc);}++ public PcodeOpAST getOp() {+ return op;+ }++ public VarnodeAST getVarnode() {+ return vn;+ }++ public ArrayList<DataVertex> getSources() {+ return sources;+ }++ /**+ * Get the depth at which the pairing occurred+ * @return depth of pairing+ */+ public int getDepthPaired() {+ int size = ngrams.size();+ if (size <= 0) {+ return 0;+ }+ DataNGram gramPaired = ngrams.get(size - 1);+ for (DataNGram gram : ngrams) {+ if (gram.hash == gramPaired.hash) {+ return gram.depth;+ }+ }+ return 0;+ }}diff --git a/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/Pinning.java b/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/Pinning.javaindex b6e255e13b..0c65fd85b3 100755--- a/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/Pinning.java+++ b/Ghidra/Features/CodeCompare/src/main/java/ghidra/features/codecompare/graphanalysis/Pinning.java@@ -908,4 +908,8 @@ public class Pinning {}return result;}++ public Map<DataVertex, DataVertex> getPinMap() {+ return pinMap;+ }}