Ghidra is a software reverse engineering (SRE) framework
GP-4045 add locking to sleigh .sla and slaspec files
dev747368c525661a4f63
diff --git a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.java b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.javaindex 8f487d6f05..e69bd9883e 100644--- a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.java+++ b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.java@@ -22,10 +22,10 @@ import docking.ActionContext;import docking.Tool;import docking.action.DockingActionIf;import docking.actions.PopupActionProvider;-import generic.jar.ResourceFile;import ghidra.app.plugin.PluginCategoryNames;import ghidra.app.plugin.core.debug.DebuggerPluginPackage;import ghidra.app.plugin.core.debug.gui.listing.DebuggerListingActionContext;+import ghidra.app.plugin.processors.sleigh.SleighLanguageDescription;import ghidra.app.services.DebuggerPlatformService;import ghidra.app.services.DebuggerTraceManagerService;import ghidra.framework.plugintool.*;@@ -202,27 +202,17 @@ public class DebuggerDisassemblerPlugin extends Plugin implements PopupActionProprotected Collection<LanguageID> getAlternativeLanguageIDs(Language language) {// One of the alternatives is the language's actual defaultLanguageDescription desc = language.getLanguageDescription();- if (!(desc instanceof SleighLanguageDescription)) {+ if (!(desc instanceof SleighLanguageDescription sld)) {return List.of();}- SleighLanguageDescription sld = (SleighLanguageDescription) desc;- ResourceFile slaFile = sld.getSlaFile();List<LanguageID> result = new ArrayList<>();LanguageService langServ = DefaultLanguageService.getLanguageService();for (LanguageDescription altDesc : langServ.getLanguageDescriptions(false)) {- if (!(altDesc instanceof SleighLanguageDescription)) {- continue;+ if (altDesc instanceof SleighLanguageDescription altSld &&+ sld.isSameSleighLanguageFile(altSld) && sld.getEndian() == altSld.getEndian()) {+ result.add(altSld.getLanguageID());}- SleighLanguageDescription altSld = (SleighLanguageDescription) altDesc;- if (!altSld.getSlaFile().equals(slaFile)) {- continue;- }- if (altSld.getEndian() != sld.getEndian()) {- // Memory endian, not necessarily instruction endian- continue;- }- result.add(altSld.getLanguageID());}return result;}diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/guest/DBTraceGuestPlatform.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/guest/DBTraceGuestPlatform.javaindex 468a8a0803..02e795ecfa 100644--- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/guest/DBTraceGuestPlatform.java+++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/guest/DBTraceGuestPlatform.java@@ -23,6 +23,7 @@ import org.apache.commons.lang3.tuple.Pair;import db.DBRecord;import generic.jar.ResourceFile;+import ghidra.app.plugin.processors.sleigh.SleighLanguageDescription;import ghidra.app.util.PseudoInstruction;import ghidra.framework.data.OpenMode;import ghidra.lifecycle.Internal;@@ -304,7 +305,7 @@ public class DBTraceGuestPlatform extends DBAnnotatedObjectprivate static ResourceFile getSlaFile(Language language) {SleighLanguageDescription desc =(SleighLanguageDescription) language.getLanguageDescription();- return desc.getSlaFile();+ return desc.getLanguageFile().getSlaFile();}@Overridediff --git a/Ghidra/Extensions/SleighDevTools/src/main/java/ghidra/app/util/disassemble/GNUExternalDisassembler.java b/Ghidra/Extensions/SleighDevTools/src/main/java/ghidra/app/util/disassemble/GNUExternalDisassembler.javaindex cbb51d82e4..b5c5aae5a7 100644--- a/Ghidra/Extensions/SleighDevTools/src/main/java/ghidra/app/util/disassemble/GNUExternalDisassembler.java+++ b/Ghidra/Extensions/SleighDevTools/src/main/java/ghidra/app/util/disassemble/GNUExternalDisassembler.java@@ -23,6 +23,7 @@ import org.jdom2.*;import org.jdom2.input.SAXBuilder;import generic.jar.ResourceFile;+import ghidra.app.plugin.processors.sleigh.SleighLanguageDescription;import ghidra.app.util.bin.ByteProvider;import ghidra.app.util.bin.MemoryByteProvider;import ghidra.framework.*;diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/help/AboutProgramPlugin.java b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/help/AboutProgramPlugin.javaindex 005178f7a1..23f140ce0f 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/help/AboutProgramPlugin.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/help/AboutProgramPlugin.java@@ -27,6 +27,7 @@ import ghidra.app.CorePluginPackage;import ghidra.app.context.ProgramActionContext;import ghidra.app.context.ProgramContextAction;import ghidra.app.plugin.PluginCategoryNames;+import ghidra.app.plugin.processors.sleigh.SleighLanguageDescription;import ghidra.app.util.GenericHelpTopics;import ghidra.app.util.HelpTopics;import ghidra.framework.main.ApplicationLevelPlugin;@@ -169,7 +170,7 @@ public class AboutProgramPlugin extends Plugin implements ApplicationLevelPluginmetadata.put("Language Spec",lDesc.getDefsFile() + (lav.isMismatch() ? lav.getVersionDisplay() : ""));metadata.put("Processor Spec", lDesc.getSpecFile().getAbsolutePath());- metadata.put("Sleigh Spec", lDesc.getSlaFile().getAbsolutePath() + "spec");+ metadata.put("Sleigh Spec", lDesc.getLanguageFile().getSlaSpecFile().getAbsolutePath());}if (lav.compilerSpec != null) {metadata.put("Compiler Spec",diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFUtil.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFUtil.javaindex 78c4f6277e..f2436a9216 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFUtil.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFUtil.java@@ -27,6 +27,7 @@ import java.util.regex.Pattern;import generic.jar.ResourceFile;import ghidra.app.cmd.comments.AppendCommentCmd;+import ghidra.app.plugin.processors.sleigh.SleighLanguageDescription;import ghidra.app.util.bin.format.dwarf.attribs.DWARFAttributeValue;import ghidra.app.util.bin.format.dwarf.attribs.DWARFNumericAttribute;import ghidra.app.util.bin.format.dwarf.expression.DWARFExpressionException;diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/task/ProgramOpener.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/task/ProgramOpener.javaindex be51454486..8f3545cbf1 100644--- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/task/ProgramOpener.java+++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/task/ProgramOpener.java@@ -170,8 +170,12 @@ public class ProgramOpener {// we don't care, the task has been cancelled}catch (LanguageNotFoundException e) {- Msg.showError(this, null, "Error Opening " + filename,- e.getMessage() + "\nPlease contact the Ghidra team for assistance.");+ String msg = e.getMessage() + "\n";+ if (e.getCause() != null) {+ msg += e.getCause().getMessage() + "\n";+ }+ msg += "Please contact the Ghidra team for assistance.";+ Msg.showError(this, null, "Error Opening " + filename, msg);}catch (Exception e) {if (domainFile.isInWritableProject() && (e instanceof IOException)) {diff --git a/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompInterface.java b/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompInterface.javaindex 25622a2c93..45fa1b9657 100644--- a/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompInterface.java+++ b/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompInterface.java@@ -25,8 +25,7 @@ import java.util.ArrayList;import generic.jar.ResourceFile;import ghidra.app.decompiler.signature.DebugSignature;import ghidra.app.decompiler.signature.SignatureResult;-import ghidra.app.plugin.processors.sleigh.SleighLanguage;-import ghidra.app.plugin.processors.sleigh.UniqueLayout;+import ghidra.app.plugin.processors.sleigh.*;import ghidra.program.model.address.*;import ghidra.program.model.lang.*;import ghidra.program.model.listing.Function;@@ -278,8 +277,7 @@ public class DecompInterface {xmlEncode.clear();dtmanage.encodeCoreTypes(xmlEncode);String coretypes = xmlEncode.toString();- SleighLanguageDescription sleighdescription =- (SleighLanguageDescription) pcodelanguage.getLanguageDescription();+ SleighLanguageDescription sleighdescription = pcodelanguage.getLanguageDescription();ResourceFile pspecfile = sleighdescription.getSpecFile();String pspecxml = fileToString(pspecfile);xmlEncode.clear();diff --git a/Ghidra/Framework/Emulation/src/test/java/ghidra/app/plugin/processors/sleigh/SleighLanguageHelper.java b/Ghidra/Framework/Emulation/src/test/java/ghidra/app/plugin/processors/sleigh/SleighLanguageHelper.javaindex 89432c7316..22593d0b02 100644--- a/Ghidra/Framework/Emulation/src/test/java/ghidra/app/plugin/processors/sleigh/SleighLanguageHelper.java+++ b/Ghidra/Framework/Emulation/src/test/java/ghidra/app/plugin/processors/sleigh/SleighLanguageHelper.java@@ -88,7 +88,8 @@ public class SleighLanguageHelper {);langDesc.setDefsFile(lDefsFile);langDesc.setSpecFile(pSpecFile);- langDesc.setSlaFile(slaFile);+ langDesc.setLanguageFile(+ SleighLanguageFile.fromSlaFilename(slaFile.getParentFile(), slaFile.getName()));MOCK_BE_64_LANGUAGE = new SleighLanguage(langDesc);return MOCK_BE_64_LANGUAGE;diff --git a/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/IOResult.java b/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/IOResult.javaindex 9a3170f9b6..0823876dea 100644--- a/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/IOResult.java+++ b/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/IOResult.java@@ -25,29 +25,78 @@ import utilities.util.reflection.ReflectionUtilities;import utility.function.Dummy;/**- * Class to pass to a thread pool that will consume all output from an external process. This is- * a {@link Runnable} that get submitted to a thread pool. This class records the data it reads+ * {@link Runnable} that will consume all text output from an {@link InputStream} tied to an+ * external processes (stdout / stderr).+ * <p>+ * The output can be inspected line-by-line by providing a string {@link Consumer}, or the entire+ * output of the process can be inspected by calling {@link #getOutput()} or+ * {@link #getOutputAsString()}.*/public class IOResult implements Runnable {public static final String THREAD_POOL_NAME = "I/O Thread Pool";- private List<String> outputLines = new ArrayList<>();+ private final List<String> outputLines;private BufferedReader commandOutput;private final Throwable inception;- private Consumer<String> consumer = Dummy.consumer();+ private final Consumer<String> consumer;+ /**+ * Creates a {@link IOResult} that consumes the specified {@link InputStream}, saving it+ * as text lines.+ *+ * @param input {@link InputStream}+ */public IOResult(InputStream input) {- this(ReflectionUtilities.createThrowableWithStackOlderThan(IOResult.class), input);+ this(input, null, true,+ ReflectionUtilities.createThrowableWithStackOlderThan(IOResult.class));}- public IOResult(Throwable inception, InputStream input) {- this.inception = inception;- commandOutput = new BufferedReader(new InputStreamReader(input));+ /**+ * Creates a {@link IOResult} that consumes the specified {@link InputStream}, saving it+ * as text lines.+ *+ * @param input {@link InputStream}+ * @param inception information about where this object was created+ */+ public IOResult(InputStream input, Throwable inception) {+ this(input, null, true, inception);+ }++ /**+ * Creates a {@link IOResult} that consumes the specified {@link InputStream}, handing each+ * line to the {@link Consumer}.+ * <p>+ * Example: {@code new IOResult(process.getInputStream(), s -> System.out.println(s), null);}+ *+ * @param input {@link InputStream}+ * @param lineConsumer {@link Consumer string consumer}+ * @param inception information about where this object was created+ */+ public IOResult(InputStream input, Consumer<String> lineConsumer, Throwable inception) {+ this(input, lineConsumer, false, inception);}- public void setConsumer(Consumer<String> consumer) {- this.consumer = consumer;+ /**+ * Creates a {@link IOResult} that consumes the specified {@link InputStream}, handing each+ * line to the {@link Consumer} and optionally storing each line for later retrieval.+ *+ * @param input {@link InputStream}+ * @param lineConsumer {@link Consumer string consumer}, optional+ * @param retainLines boolean flag, if true, the contents read from the InputStream will be+ * available via {@link #getOutput()} and {@link #getOutputAsString()}+ * @param inception information about where this object was created+ */+ public IOResult(InputStream input, Consumer<String> lineConsumer, boolean retainLines,+ Throwable inception) {+ this.outputLines = retainLines ? new ArrayList<>() : List.of();+ if (retainLines) {+ lineConsumer = Dummy.ifNull(lineConsumer).andThen(outputLines::add);+ }+ this.consumer = lineConsumer;+ this.inception = inception;++ commandOutput = new BufferedReader(new InputStreamReader(input));}public String getOutputAsString() {@@ -68,8 +117,7 @@ public class IOResult implements Runnable {try {while ((line = commandOutput.readLine()) != null) {- consumer.accept(line);- outputLines.add(line);+ consumer.accept(line); // this both adds to outputLines and calls the upstream consumer}}catch (Exception e) {diff --git a/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/ProcessConsumer.java b/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/ProcessConsumer.javaindex 0563259bf6..86eddb98cd 100644--- a/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/ProcessConsumer.java+++ b/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/ProcessConsumer.java@@ -53,8 +53,7 @@ public class ProcessConsumer {* @param lineConsumer the line consumer; may be null* @return the future that will be complete when all lines are read*/- public static Future<IOResult> consume(InputStream is,- Consumer<String> lineConsumer) {+ public static Future<IOResult> consume(InputStream is, Consumer<String> lineConsumer) {lineConsumer = Dummy.ifNull(lineConsumer);@@ -62,9 +61,38 @@ public class ProcessConsumer {ReflectionUtilities.createThrowableWithStackOlderThan(ProcessConsumer.class));GThreadPool pool = GThreadPool.getSharedThreadPool(IOResult.THREAD_POOL_NAME);- IOResult runnable = new IOResult(inception, is);- runnable.setConsumer(lineConsumer);+ IOResult runnable = new IOResult(is, lineConsumer, true, inception);Future<IOResult> future = pool.submit(runnable, runnable);return future;}++ /**+ * Reads the given input stream line-by-line, calling the given consumer. When the+ * InputStream reaches EOF, a final {@code null} will be sent to the line consumer.+ * <p>+ * The Inputstream is consumed via a Thread created just for this setup.+ *+ * @param is the input stream+ * @param lineConsumer the line consumer; may be null+ * @param processName descriptive name of process being monitored+ */+ public static void monitorAndSignalEof(InputStream is, Consumer<String> lineConsumer,+ String processName) {++ Throwable inception = ReflectionUtilities.filterJavaThrowable(+ ReflectionUtilities.createThrowableWithStackOlderThan(ProcessConsumer.class));++ IOResult stdoutReader = new IOResult(is, lineConsumer, false, inception);++ Runnable threadRunnable = lineConsumer != null // change mode if null+ ? () -> {+ stdoutReader.run();+ lineConsumer.accept(null); // signal that eof was reached+ }+ : stdoutReader;++ Thread t = new Thread(threadRunnable, "IO Thread for " + processName);+ t.setDaemon(true);+ t.start();+ }}diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageExtraMethods.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileException.javasimilarity index 57%rename from Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageExtraMethods.javarename to Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileException.javaindex c80872e4c2..130dffafe4 100644--- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageExtraMethods.java+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileException.java@@ -1,27 +1,31 @@/* ###* IP: GHIDRA- * REVIEWED: YES** Licensed under the Apache License, Version 2.0 (the "License");* you may not use this file except in compliance with the License.* You may obtain a copy of the License at- *+ ** http://www.apache.org/licenses/LICENSE-2.0- *+ ** Unless required by applicable law or agreed to in writing, software* distributed under the License is distributed on an "AS IS" BASIS,* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.* See the License for the specific language governing permissions and* limitations under the License.*/-//package ghidra.app.plugin.processors.sleigh;-//-//import ghidra.app.plugin.processors.sleigh.symbol.SymbolTable;-//import ghidra.program.model.address.AddressSpace;-//import ghidra.program.model.lang.Language;-//-//public interface SleighLanguage extends Language {-// public AddressSpace getDefaultSpace();-// public SymbolTable getSymbolTable();-// public DecisionNode getRootDecisionNode();-//}+package ghidra.app.plugin.processors.sleigh;++/**+ * Thrown when error concerning a sleigh file is encountered+ */+public class SleighFileException extends SleighException {++ public SleighFileException(String message) {+ super(message);+ }++ public SleighFileException(String message, Throwable e) {+ super(message, e);+ }++}diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileLockException.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileLockException.javanew file mode 100644index 0000000000..2fde6dfa95--- /dev/null+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileLockException.java@@ -0,0 +1,28 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.app.plugin.processors.sleigh;++public class SleighFileLockException extends SleighFileException {++ public SleighFileLockException(String message) {+ super(message);+ }++ public SleighFileLockException(String message, Throwable e) {+ super(message, e);+ }++}diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguage.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguage.javaindex aa5a38ce71..5c05cd94bc 100644--- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguage.java+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguage.java@@ -22,10 +22,10 @@ import java.math.BigInteger;import java.util.*;import java.util.Map.Entry;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.atomic.AtomicLong;import java.util.regex.Matcher;import java.util.regex.Pattern;-import org.antlr.runtime.RecognitionException;import org.xml.sax.*;import generic.jar.ResourceFile;@@ -36,7 +36,6 @@ import ghidra.app.plugin.processors.sleigh.expression.PatternValue;import ghidra.app.plugin.processors.sleigh.symbol.*;import ghidra.framework.Application;import ghidra.pcode.utils.SlaFormat;-import ghidra.pcodeCPort.slgh_compile.SleighCompileLauncher;import ghidra.program.model.address.*;import ghidra.program.model.lang.*;import ghidra.program.model.listing.DefaultProgramContext;@@ -44,10 +43,11 @@ import ghidra.program.model.mem.MemBuffer;import ghidra.program.model.mem.MemoryAccessException;import ghidra.program.model.pcode.*;import ghidra.program.model.util.ProcessorSymbolType;-import ghidra.sleigh.grammar.SleighPreprocessor;import ghidra.sleigh.grammar.SourceFileIndexer;import ghidra.util.*;import ghidra.util.exception.InvalidInputException;+import ghidra.util.exception.TimeoutException;+import ghidra.util.task.PreserveStateWrappingTaskMonitor;import ghidra.util.task.TaskMonitor;import ghidra.util.xml.SpecXmlUtils;import ghidra.xml.*;@@ -70,7 +70,7 @@ public class SleighLanguage implements Language {private int numSections = 0; // Number of named sections for this languageprivate int alignment = 1;private int defaultPointerWordSize = 1; // Default wordsize to send down with pointer data-types- private SleighLanguageDescription description;+ private final SleighLanguageDescription description;private ParallelInstructionLanguageHelper parallelHelper;private SourceFileIndexer indexer; //used to provide source file info for constructors@@ -99,11 +99,16 @@ public class SleighLanguage implements Language {private AddressSpace default_space;private List<ContextSetting> ctxsetting = new ArrayList<>();private LinkedHashMap<String, String> properties = new LinkedHashMap<>();- SortedMap<String, ManualEntry> manual = null;+ private SortedMap<String, ManualEntry> manual = null;- SleighLanguage(SleighLanguageDescription description)- throws DecoderException, SAXException, IOException {- initialize(description);+ SleighLanguage(SleighLanguageDescription description) throws SleighException {+ this(description, TaskMonitor.DUMMY);+ }++ SleighLanguage(SleighLanguageDescription description, TaskMonitor monitor)+ throws SleighException {+ this.description = description;+ initialize(false, monitor);}private void addAdditionInject(InjectPayloadSleigh payload) {@@ -113,37 +118,68 @@ public class SleighLanguage implements Language {additionalInject.add(payload);}- private void initialize(SleighLanguageDescription langDescription)- throws DecoderException, SAXException, IOException {+ private void initialize(boolean forceCompile, TaskMonitor monitor) throws SleighException {+ long startTS = System.currentTimeMillis();+this.defaultSymbols = new ArrayList<>();this.defaultMemoryBlocks = new MemoryBlockDefinition[0];this.compilerSpecDescriptions = new LinkedHashMap<>();- for (CompilerSpecDescription compilerSpecDescription : langDescription+ for (CompilerSpecDescription compilerSpecDescription : description.getCompatibleCompilerSpecDescriptions()) {this.compilerSpecDescriptions.put(compilerSpecDescription.getCompilerSpecID(),(SleighCompilerSpecDescription) compilerSpecDescription);}compilerSpecs = new HashMap<>();- this.description = langDescription;additionalInject = null;- SleighLanguageValidator.validatePspecFile(langDescription.getSpecFile());+ SleighLanguageValidator.validatePspecFile(description.getSpecFile());readInitialDescription();- // should addressFactory and registers initialization be done at- // construction time?+ // should addressFactory and registers initialization be done at construction time?// for now we'll assume yes.contextcache = new ContextCache();-- ResourceFile slaFile = langDescription.getSlaFile();- if (!slaFile.exists() ||- (slaFile.canWrite() && (isSLAWrongVersion(slaFile) || isSLAStale(slaFile)))) {- reloadLanguage(TaskMonitor.DUMMY, true);++ SleighLanguageFile langFile = description.getLanguageFile();+ if (!langFile.getSlaSpecFile().exists()) {+ throw new SleighFileException("Missing slaspec: " + langFile.getSlaSpecFile());+ }++ // check .sla file freshness inside lock, and recompile if necessary before releasing lock.+ // if can't lock, it's single jar mode and we can't recompile anyways+ AtomicLong lockElapsed = new AtomicLong();+ if (langFile.canLock()) {+ try (PreserveStateWrappingTaskMonitor tm =+ new PreserveStateWrappingTaskMonitor(monitor)) {+ tm.setCancelEnabled(true);+ tm.setShowProgressValue(true);+ langFile.withLock(SleighLanguageProvider.LANGUAGE_LOCK_TIMEOUT, tm, () -> {+ tm.setCancelEnabled(false);+ long lockStartTS = System.currentTimeMillis();+ if (forceCompile || langFile.needsCompilation(SlaFormat.FORMAT_VERSION)) {+ langFile.compileSlaFile(monitor);+ }+ lockElapsed.set(System.currentTimeMillis() - lockStartTS);+ });+ }+ catch (TimeoutException e) {+ throw new SleighFileLockException(+ "Timeout waiting for Sleigh language file lock: %s"+ .formatted(langFile.getSlaFile()),+ e);+ }+ catch (IOException e) {+ throw new SleighFileException(+ "Error locking Sleigh language file %s".formatted(langFile.getSlaFile()), e);+ }}-+// Read in the sleigh specification- PackedDecode decoder = SlaFormat.buildDecoder(slaFile);- decode(decoder);+ try (PackedDecode decoder = SlaFormat.buildDecoder(langFile.getSlaFile())) {+ decode(decoder);+ }+ catch (IOException | DecoderException e) {+ throw new SleighException("Error decoding", e);+ }registerBuilder = new RegisterBuilder();loadRegisters(registerBuilder);@@ -154,6 +190,10 @@ public class SleighLanguage implements Language {instructProtoMap = new ConcurrentHashMap<>();initParallelHelper();++ long initElapsed = System.currentTimeMillis() - startTS;+ Msg.debug(this, "Took %dms (%dms inside lock) to initialize language %s"+ .formatted(initElapsed, lockElapsed.get(), langFile));}private void buildVolatileSymbolAddresses() {@@ -165,37 +205,6 @@ public class SleighLanguage implements Language {}}- private boolean isSLAWrongVersion(ResourceFile slaFile) {- try (InputStream stream = slaFile.getInputStream()) {- return !SlaFormat.isSlaFormat(stream);- }- catch (Exception e) {- return true;- }- }-- private boolean isSLAStale(ResourceFile slaFile) {- String slafilename = slaFile.getName();- int index = slafilename.lastIndexOf('.');- String slabase = slafilename.substring(0, index);- String slaspecfilename = slabase + ".slaspec";- ResourceFile slaspecFile = new ResourceFile(slaFile.getParentFile(), slaspecfilename);-- File resourceAsFile = slaspecFile.getFile(true);- SleighPreprocessor preprocessor =- new SleighPreprocessor(new ModuleDefinitionsAdapter(), resourceAsFile);- long sourceTimestamp = Long.MAX_VALUE;- try {- sourceTimestamp = preprocessor.scanForTimestamp();- }- catch (Exception e) {- // squash the error because we will force recompilation and errors- // will propagate elsewhere- }- long compiledTimestamp = slaFile.lastModified();- return (sourceTimestamp > compiledTimestamp);- }-/*** Returns the unique base offset from which additional temporary variables* may be created.@@ -417,79 +426,11 @@ public class SleighLanguage implements Language {@Overridepublic void reloadLanguage(TaskMonitor monitor) throws IOException {- reloadLanguage(monitor, false);- }-- private void reloadLanguage(TaskMonitor monitor, boolean calledFromInitialize)- throws IOException {- if (monitor == null) {- monitor = TaskMonitor.DUMMY;- }- monitor.setMessage("Compiling Language File...");-- ResourceFile slaFile = description.getSlaFile();- String slaName = slaFile.getName();- int index = slaName.lastIndexOf('.');- String specName = slaName.substring(0, index);- String languageName = specName + ".slaspec";- ResourceFile languageFile = new ResourceFile(slaFile.getParentFile(), languageName);-- // see gradle/processorUtils.gradle for sleighArgs.txt generation- ResourceFile sleighArgsFile = null;- ResourceFile languageModule = Application.getModuleContainingResourceFile(languageFile);- if (languageModule != null) {- if (SystemUtilities.isInReleaseMode()) {- sleighArgsFile = new ResourceFile(languageModule, "data/sleighArgs.txt");- }- else {- sleighArgsFile = new ResourceFile(languageModule, "build/tmp/sleighArgs.txt");- }- }-- String[] args;- if (sleighArgsFile != null && sleighArgsFile.isFile()) {- String baseDir = Application.getInstallationDirectory()- .getAbsolutePath()- .replace(File.separatorChar, '/');- if (!baseDir.endsWith("/")) {- baseDir += "/";- }- args = new String[] { "-DBaseDir=" + baseDir, "-i", sleighArgsFile.getAbsolutePath(),- languageFile.getAbsolutePath(), description.getSlaFile().getAbsolutePath() };- }- else {- args = new String[] { languageFile.getAbsolutePath(),- description.getSlaFile().getAbsolutePath() };- }-try {- StringBuilder buf = new StringBuilder();- for (String str : args) {- buf.append(str);- buf.append(" ");- }- Msg.debug(this, "Sleigh compile: " + buf);- int returnCode = SleighCompileLauncher.runMain(args);- if (returnCode != 0) {- throw new SleighException("Errors compiling " + languageFile.getAbsolutePath() +- " -- please check log messages for details");- }- }- catch (RecognitionException e) {- throw new IOException("RecognitionException error recompiling: " + e.getMessage());+ initialize(true, TaskMonitor.dummyIfNull(monitor));}-- if (!calledFromInitialize) {- monitor.setMessage("Reloading Language...");- try {- initialize(description);- }- catch (DecoderException e) {- throw new IOException(e.getMessage());- }- catch (SAXException e) {- throw new IOException(e.getMessage());- }+ catch (SleighException e) {+ throw new IOException("Failed to reload Sleigh language", e);}}@@ -517,26 +458,33 @@ public class SleighLanguage implements Language {}};- private void readInitialDescription() throws SAXException, IOException {- ResourceFile specFile = description.getSpecFile();- XmlPullParser parser = XmlPullParserFactory.create(specFile, SPEC_ERR_HANDLER, false);+ private void readInitialDescription() throws SleighException {try {- XmlElement nextElement = parser.peek();- while (nextElement != null && !nextElement.getName().equals("segmented_address")) {- parser.next(); // skip element- nextElement = parser.peek();- }- if (nextElement != null) {- XmlElement element = parser.start(); // segmented_address element- segmentedspace = element.getAttribute("space");- segmentType = element.getAttribute("type");- if (segmentType == null) {- segmentType = "";+ ResourceFile specFile = description.getSpecFile();+ XmlPullParser parser = XmlPullParserFactory.create(specFile, SPEC_ERR_HANDLER, false);+ try {+ XmlElement nextElement = parser.peek();+ while (nextElement != null && !nextElement.getName().equals("segmented_address")) {+ parser.next(); // skip element+ nextElement = parser.peek();+ }+ if (nextElement != null) {+ XmlElement element = parser.start(); // segmented_address element+ segmentedspace = element.getAttribute("space");+ segmentType = element.getAttribute("type");+ if (segmentType == null) {+ segmentType = "";+ }}}+ finally {+ parser.dispose();+ }}- finally {- parser.dispose();+ catch (SAXException | IOException e) {+ throw new SleighException(+ "Error reading initial description - language probably did not compile properly",+ e);}}@@ -864,19 +812,26 @@ public class SleighLanguage implements Language {parser.end(el);}- private void readRemainingSpecification() throws SAXException, IOException {- ResourceFile specFile = description.getSpecFile();- XmlPullParser parser = XmlPullParserFactory.create(specFile, SPEC_ERR_HANDLER, false);+ private void readRemainingSpecification() throws SleighException {try {- read(parser);- }- catch (XmlParseException e) {- Msg.error(this, "Failed to parse Sleigh Specification (" + specFile.getName() + "): " +- e.getMessage());+ ResourceFile specFile = description.getSpecFile();+ XmlPullParser parser = XmlPullParserFactory.create(specFile, SPEC_ERR_HANDLER, false);+ try {+ read(parser);+ }+ catch (XmlParseException e) {+ Msg.error(this, "Failed to parse Sleigh Specification (" + specFile.getName() ++ "): " + e.getMessage());+ }+ finally {+ parser.dispose();+ }}- finally {- parser.dispose();+ catch (SAXException | IOException e) {+ throw new SleighException(+ "Error reading remaining spec - language probably did not compile properly", e);}+}private void decode(Decoder decoder) throws DecoderException {@@ -1163,7 +1118,7 @@ public class SleighLanguage implements Language {}@Override- public LanguageDescription getLanguageDescription() {+ public SleighLanguageDescription getLanguageDescription() {return description;}@@ -1503,8 +1458,7 @@ public class SleighLanguage implements Language {}encoder.closeElement(ElementId.ELEM_SPACES);- SleighLanguageDescription sleighDescription =- (SleighLanguageDescription) getLanguageDescription();+ SleighLanguageDescription sleighDescription = getLanguageDescription();Set<String> truncatedSpaceNames = sleighDescription.getTruncatedSpaceNames();if (!truncatedSpaceNames.isEmpty()) {for (String spaceName : truncatedSpaceNames) {diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/SleighLanguageDescription.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageDescription.javasimilarity index 72%rename from Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/SleighLanguageDescription.javarename to Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageDescription.javaindex 75b0a0adb9..63d2973b3e 100644--- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/SleighLanguageDescription.java+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageDescription.java@@ -4,30 +4,31 @@* Licensed under the Apache License, Version 2.0 (the "License");* you may not use this file except in compliance with the License.* You may obtain a copy of the License at- *+ ** http://www.apache.org/licenses/LICENSE-2.0- *+ ** Unless required by applicable law or agreed to in writing, software* distributed under the License is distributed on an "AS IS" BASIS,* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.* See the License for the specific language governing permissions and* limitations under the License.*/-package ghidra.program.model.lang;--import generic.jar.ResourceFile;+package ghidra.app.plugin.processors.sleigh;import java.util.*;+import generic.jar.ResourceFile;+import ghidra.program.model.lang.*;+/*** Class for holding Language identifiers*/public class SleighLanguageDescription extends BasicLanguageDescription {private ResourceFile defsFile; // defs file- private ResourceFile specFile; // specification file- private ResourceFile slaFile; // just cramming this in here until major cleanup+ private ResourceFile specFile; // pspec specification fileprivate ResourceFile manualIndexFile; // the manual index file+ private SleighLanguageFile languageFile; // sla, slaspec fileprivate Map<String, Integer> truncatedSpaceMap;@@ -36,8 +37,8 @@ public class SleighLanguageDescription extends BasicLanguageDescription {* @param id the name of the language* @param description language description text* @param processor processor name/family- * @param endian data endianness- * @param instructionEndian instruction endianness+ * @param endian data endianess+ * @param instructionEndian instruction endianess* @param size processor size* @param variant processor variant name* @param version the major version of the language.@@ -54,19 +55,15 @@ public class SleighLanguageDescription extends BasicLanguageDescription {Map<String, List<String>> externalNames) {super(id, processor, endian, instructionEndian, size, variant, description, version,minorVersion, deprecated, compilerSpecDescriptions, externalNames);- this.specFile = null;- this.slaFile = null;- this.manualIndexFile = null;this.truncatedSpaceMap = spaceTruncations;}/*** @return set of address space names which have been identified for truncation*/- @SuppressWarnings("unchecked")public Set<String> getTruncatedSpaceNames() {if (truncatedSpaceMap == null) {- return Collections.EMPTY_SET;+ return Set.of();}return truncatedSpaceMap.keySet();}@@ -107,33 +104,37 @@ public class SleighLanguageDescription extends BasicLanguageDescription {* Set the (optional) specification file associated with this language** @param specFile- * the specFile to associate with this description.+ * the specFile (.pspec) to associate with this description.*/public void setSpecFile(ResourceFile specFile) {this.specFile = specFile;}/**- * Get the specification file (if it exists)+ * Get the specification (.pspec) file (if it exists)*- * @return specification file+ * @return specification file (.pspec)*/public ResourceFile getSpecFile() {return specFile;}/**- * @param slaFile+ * Sets the {@link SleighLanguageFile} which represents the .sla and .slaspec files.+ *+ * @param langFile {@link SleighLanguageFile} which represents the .sla and .slaspec files*/- public void setSlaFile(ResourceFile slaFile) {- this.slaFile = slaFile;+ void setLanguageFile(SleighLanguageFile langFile) {+ this.languageFile = langFile;}/**- * @return+ * Returns the {@link SleighLanguageFile} which represents the .sla and .slaspec files.+ *+ * @return {@link SleighLanguageFile} which represents the .sla and .slaspec files*/- public ResourceFile getSlaFile() {- return slaFile;+ public SleighLanguageFile getLanguageFile() {+ return languageFile;}public ResourceFile getManualIndexFile() {@@ -143,4 +144,16 @@ public class SleighLanguageDescription extends BasicLanguageDescription {public void setManualIndexFile(ResourceFile manualIndexFile) {this.manualIndexFile = manualIndexFile;}++ /**+ * Tests if two Sleigh languages are based on the same .sla file.+ *+ * @param other {@link SleighLanguageDescription}+ * @return true if the other {@link SleighLanguageDescription} uses the same .sla file+ */+ public boolean isSameSleighLanguageFile(SleighLanguageDescription other) {+ return other != null &&+ languageFile.getSlaFile().equals(other.getLanguageFile().getSlaFile());+ }+}diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageFile.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageFile.javanew file mode 100644index 0000000000..775fed826a--- /dev/null+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageFile.java@@ -0,0 +1,533 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.app.plugin.processors.sleigh;++import static utilities.util.FileUtilities.*;++import java.io.*;+import java.nio.channels.FileLock;+import java.nio.charset.StandardCharsets;+import java.time.Duration;+import java.util.ArrayList;+import java.util.List;+import java.util.regex.Pattern;++import org.antlr.runtime.RecognitionException;+import org.apache.commons.io.FilenameUtils;++import generic.jar.ResourceFile;+import ghidra.framework.Application;+import ghidra.pcode.utils.SlaFormat;+import ghidra.pcodeCPort.slgh_compile.SleighCompile;+import ghidra.pcodeCPort.slgh_compile.SleighCompileOptions;+import ghidra.sleigh.grammar.SleighPreprocessor;+import ghidra.util.SystemUtilities;+import ghidra.util.exception.TimeoutException;+import ghidra.util.task.TaskMonitor;+import utilities.util.FileResolutionResult;+import utilities.util.FileUtilities;++/**+ * Represents a Sleigh .sla and .slaspec file, and a way to lock them to ensure exclusive access+ * while checking / updating the files.+ */+public class SleighLanguageFile {+ public static final String SLASPEC_EXT = ".slaspec";+ public static final String SLA_EXT = ".sla";++ /**+ * Finds a sleigh language file, using search rules specific to sleigh.+ * <p>+ * If file is not found in the specific base directory, the entire app install will be searched+ * for a matching file.+ *+ * @param dir base directory where files are typically located+ * @param filename name of file, with or without the extension+ * @param expectedExtension extension of the specific type of sleigh file, leading dot required.+ * Typically ".sla", ".slaspec", ".pspec", etc.+ * @return ResourceFile that exists, never null+ * @throws SleighFileException if file is not found or has bad case matching+ */+ public static ResourceFile getLanguageResourceFile(ResourceFile dir, String filename,+ String expectedExtension) throws SleighFileException {+ ResourceFile f = findFile(dir, filename, expectedExtension);+ if (f == null) {+ f = new ResourceFile(dir, filename);+ throw new SleighFileException(+ "Missing sleigh file(%s): %s".formatted(expectedExtension, f.getAbsolutePath()));+ }++ FileResolutionResult result = existsAndIsCaseDependent(f);+ if (!result.isOk()) {+ throw new SleighFileException("Sleigh file %s is not properly case dependent: %s"+ .formatted(f.getAbsolutePath(), result.getMessage()));+ }+ return f;+ }++ /**+ * Creates a {@link SleighLanguageFile} instance using the language directory and sla filename+ * to bootstrap the information about the sla file, slaspec file and lock file.+ * <p>+ * NOTE: if the sla/slaspec are not found in the specified language directory, the entire+ * application will be searched for the slaspec.+ *+ * @param dir {@link ResourceFile} language directory that should contain the sla or slaspec+ * file+ * @param slaFilename name of the sla file (typically from the ldefs xml value), with optional+ * .sla file extension+ * @return {@link SleighLanguageFile}, never null+ * @throws SleighFileException if sla and slaspec file can not be found+ */+ public static SleighLanguageFile fromSlaFilename(ResourceFile dir, String slaFilename)+ throws SleighFileException {+ String baseName = slaFilename.endsWith(SLA_EXT)+ ? FilenameUtils.removeExtension(slaFilename)+ : slaFilename;++ ResourceFile slaSpecRFile;+ ResourceFile slaRFile;+ try {+ // find the slaspec and construct the sla filename from the slaspec's location+ slaSpecRFile = getLanguageResourceFile(dir, baseName + SLASPEC_EXT, SLASPEC_EXT);+ slaRFile = new ResourceFile(slaSpecRFile.getParentFile(),+ FilenameUtils.removeExtension(slaSpecRFile.getName()) + SLA_EXT);+ }+ catch (SleighFileException e) {+ try {+ // if slaspec is not found, fall back and search for .sla file.+ // if sla found, assume slaspec should be co-located there.+ // This results in a SleighLanguageFile instance that allows+ // the language description to be created, but the SleighLanguage+ // will fail to initialize because of the missing .slaspec+ slaRFile = getLanguageResourceFile(dir, baseName + SLA_EXT, SLA_EXT);+ slaSpecRFile = new ResourceFile(slaRFile.getParentFile(),+ FilenameUtils.removeExtension(slaRFile.getName()) + SLASPEC_EXT);+ }+ catch (SleighFileException e2) {+ throw e; // throw original exception, not this one+ }+ }+ if (slaSpecRFile.getFile(false) == null) {+ // single jar mode, no locking, no compiling of sla file possible++ if (!slaRFile.exists()) {+ throw new SleighFileException(+ "Missing sleigh sla file: " + slaRFile.getAbsolutePath());+ }+ return new SleighLanguageFile(slaRFile, slaSpecRFile, null);+ }++ File lockFile =+ new ResourceFile(slaRFile.getParentFile(), slaRFile.getName() + ".lock").getFile(false);++ return new SleighLanguageFile(slaRFile, slaSpecRFile, lockFile);+ }++ /**+ * Creates a {@link SleighLanguageFile} instance using the language directory and Sleigh+ * language file name to bootstrap the information about the slaspec file and lock file.+ * <p>+ * The .slaspec file will be found in the location indicated by the slaFilename.+ * <p>+ * The actual .sla file will be private to the user and located under the user's+ * .ghidra config directory. (or if single-jar mode, .sla will remain in the jar)+ *+ * @param dir {@link ResourceFile} language directory containing the slaspec file+ * @param slaFilename name of the sla file (typically from the ldefs xml value), with optional+ * .sla file extension+ * @return {@link SleighLanguageFile}, never null+ * @throws SleighFileException if slaspec is not found+ */+ public static SleighLanguageFile fromSlaFilename_UserDir(ResourceFile dir, String slaFilename)+ throws SleighFileException {++ String baseName = slaFilename.endsWith(SLA_EXT)+ ? FilenameUtils.removeExtension(slaFilename)+ : slaFilename;++ ResourceFile slaSpecRFile =+ getLanguageResourceFile(dir, baseName + SLASPEC_EXT, SLASPEC_EXT);+ if (slaSpecRFile.getFile(false) == null) {+ // single jar mode, no locking, no compiling of sla file possible+ ResourceFile slaRFile = new ResourceFile(slaSpecRFile.getParentFile(),+ FilenameUtils.removeExtension(slaSpecRFile.getName()) + SLA_EXT);+ if (!slaRFile.exists()) {+ throw new SleighFileException(+ "Missing sleigh sla file: " + slaRFile.getAbsolutePath());+ }+ return new SleighLanguageFile(slaRFile, slaSpecRFile, null);+ }++ File sleighUserDir =+ new File(Application.getApplicationLayout().getUserSettingsDir(), "sleigh");+ if (!FileUtilities.mkdirs(sleighUserDir)) {+ throw new SleighFileException("Bad user settings /sleigh directory: " + sleighUserDir);+ }++ File slaFile = new File(sleighUserDir,+ FilenameUtils.removeExtension(slaSpecRFile.getName()) + SLA_EXT);++ File lockFile = new File(sleighUserDir, slaFile.getName() + ".lock");+ return new SleighLanguageFile(new ResourceFile(slaFile), slaSpecRFile, lockFile);+ }++ private final ResourceFile slaFile;+ private final ResourceFile slaSpecFile;+ private final File lockFile;++ private SleighLanguageFile(ResourceFile slaFile, ResourceFile slaSpecFile, File lockFile) {+ this.slaFile = slaFile;+ this.slaSpecFile = slaSpecFile;+ this.lockFile = lockFile;+ }++ /**+ * Returns the path of this language's .sla file.+ *+ * @return path of the .sla file+ */+ public ResourceFile getSlaFile() {+ return slaFile;+ }++ /**+ * Returns the path of the .slaspec file.+ *+ * @return .slaspec file+ */+ public ResourceFile getSlaSpecFile() {+ return slaSpecFile;+ }++ @Override+ public String toString() {+ return "%s -> %s".formatted(slaSpecFile.getAbsolutePath(), slaFile.getAbsolutePath());+ }++ /**+ * Returns true if this language's files can be locked, or false if they can't be locked+ * (embedded in a .jar file).+ *+ * @return true if this language's files can be locked+ */+ public boolean canLock() {+ return lockFile != null;+ }++ /**+ * Executes a runnable while a lock file is being held.+ *+ * @param <E> Exception type thrown by the runnable+ * @param timeout maximum amount of time to wait to acquire the lock file. This timeout does+ * not apply to the Runnable that is executed once the lock is acquired.+ * @param monitor {@link TaskMonitor} that will be updated with lock file acquire attempt info+ * @param r a runnable that can throw a SleighException+ * @throws E if runnable throws E+ * @throws IOException IO error acquiring the lock file+ * @throws TimeoutException if lock times out+ */+ <E extends Throwable> void withLock(Duration timeout, TaskMonitor monitor,+ CheckedRunnable<E> r) throws E, IOException, TimeoutException {++ if (lockFile == null) {+ throw new IOException("Unable to lock language, missing lock file");+ }++ long timeoutMS = timeout.toMillis();+ long startts = System.currentTimeMillis();+ long maxts = startts + timeoutMS;+ long sleepMS = Math.min(timeoutMS, 100); // 100ms is largest sleep-per-retry interval+ long lockerPid = -1;++ monitor.initialize(timeoutMS / 1000, "Locking Sleigh language file");+ monitor.setProgress(timeoutMS / 1000); // run the progress meter as count down from max to 0++ try (RandomAccessFile raf = new RandomAccessFile(lockFile, "rw")) {+ while (!monitor.isCancelled()) {+ // Lock an unused/non-existent portion of the file to avoid read/write errors+ // by other processes on Windows jvms+ try (FileLock lock = raf.getChannel().tryLock(LOCKFILE_LOCK_OFFSET, 1, false)) {+ if (lock != null) {+ writeLockerInfo(raf);+ r.run();+ raf.setLength(0);+ return;+ }+ }++ // failed to acquire lock, sleep and try again+ long remaining = maxts - System.currentTimeMillis();+ if (remaining < 0) {+ // failed to acquire lock within allowed time, give up+ break;+ }++ lockerPid = tryReadLockerInfo(raf);+ monitor.setMessage("Waiting on pid [%s] for Sleigh language file lock"+ .formatted(lockerPid != -1 ? Long.toString(lockerPid) : "unknown"));+ monitor.setProgress(remaining / 1000);++ Thread.sleep(sleepMS);+ }+ throw new TimeoutException(+ "Timeout when trying to lock Sleigh language file [%s], locker's pid: [%s]"+ .formatted(lockFile,+ lockerPid != -1 ? Long.toString(lockerPid) : "unknown"));+ }+ catch (IOException | InterruptedException e) {+ throw new IOException("Error locking Sleigh language file [%s]".formatted(lockFile), e);+ }+ }++ /**+ * Checks if the sla file needs to be compiled/re-compiled.+ * <p>+ * Conditions: missing .sla, .sla is older than the .slaspec, or the sla format version value+ * inside the existing .sla file does not match the current sla format version.+ * <p>+ * NOTE: this should only be called when holding the lock with+ * {@link #withLock(Duration, TaskMonitor, CheckedRunnable)}+ *+ * @param requiredSlaFormatVersion required sla format version+ * @return true if the .sla file needs to be compiled/recompiled+ */+ public boolean needsCompilation(int requiredSlaFormatVersion) {+ return !slaFile.exists() || isSlaFileStale() || getSlaVersion() != requiredSlaFormatVersion;+ }++ /**+ * Returns true if the slaspec file (or any included sinc files) is newer than the current+ * sla file, indicating that the sla file should be recompiled.+ * <p>+ * Returns false if the slaspec or sla files are embedded in a jar (eg. single-jar mode).+ * <p>+ * NOTE: call this when holding the lock using {@link #withLock(Duration, TaskMonitor, CheckedRunnable)}+ *+ * @return true if slaspec file is newer than the sla file+ */+ public boolean isSlaFileStale() {+ // NOTE: SleighPreprocessor doesn't use ResourceFiles, so any 'include' directives processed+ // by it won't use ResourceFile.getFile()+ File f = slaSpecFile.getFile(false);+ if (f == null) {+ // if the slaspec file is embedded in a jar, always assume the sla file is correct+ return false;+ }+ long slaSpecLastMod;+ try {+ SleighPreprocessor preprocessor =+ new SleighPreprocessor(new ModuleDefinitionsAdapter(), f);+ slaSpecLastMod = preprocessor.scanForTimestamp();+ }+ catch (Exception e) {+ // slaSpecLastMod will be max_value which will force recompilation, error parsing+ // will be handled elsewhere+ slaSpecLastMod = Long.MAX_VALUE;+ }+ long slaLastMod = slaFile.lastModified(); // will be 0 if does not exist+ return slaLastMod == 0 || slaSpecLastMod > slaLastMod;+ }++ /**+ * Returns the format version number embedded in the compiled sla file.+ * <p>+ * NOTE: this should only be called when holding the lock with+ * {@link #withLock(Duration, TaskMonitor, CheckedRunnable)}+ *+ * @return format version number embedded in the sla file, or -1 if error reading or+ * file doesn't exist+ */+ public int getSlaVersion() {+ try (InputStream stream = slaFile.getInputStream()) {+ return SlaFormat.getSlaFormat(stream);+ }+ catch (Exception e) {+ return -1;+ }+ }++ /**+ * Compiles the slaspec file and replaces the sla file with the newly compiled sleigh output.+ * <p>+ * NOTE: this should only be called when holding the lock with+ * {@link #withLock(Duration, TaskMonitor, CheckedRunnable)}+ *+ * @param monitor {@link TaskMonitor}+ * @throws SleighException if error occurs during compilation+ */+ public void compileSlaFile(TaskMonitor monitor) throws SleighException {+ monitor.setMessage("Compiling Language File...");+ monitor.setIndeterminate(true);++ // see gradle/processorUtils.gradle for sleighArgs.txt generation+ String baseDir = FilenameUtils+ .separatorsToUnix(Application.getInstallationDirectory().getAbsolutePath());+ if (!baseDir.endsWith("/")) {+ baseDir += "/";+ }++ ResourceFile sleighArgsFile = getSleighArgsForSlaSpec(slaSpecFile);+ SleighCompileOptions compileOptions = sleighArgsFile != null && sleighArgsFile.exists()+ ? SleighCompileOptions.fromFile(sleighArgsFile.getFile(true))+ : new SleighCompileOptions();++ compileOptions.addPreprocessorMacroDefinition("BaseDir", baseDir);++ File inputFile = slaSpecFile.getFile(true);++ File destFile = slaFile.getFile(true);+ long mypid = ProcessHandle.current().pid();+ File destTmpFile =+ new File(destFile.getParentFile(), destFile.getName() + ".%d.tmp".formatted(mypid));++ SleighCompile compiler = new SleighCompile();+ compiler.setOptions(compileOptions);++ try {+ int returnCode = compiler.run_compilation(inputFile.getPath(), destTmpFile.getPath());+ if (returnCode != 0) {+ destTmpFile.delete();+ throw new SleighException(+ "Errors compiling %s -- please check log messages for details"+ .formatted(slaSpecFile));+ }+ if (!destTmpFile.renameTo(destFile)) {+ // atomically renaming the tmp file to replace the existing dest file will+ // succeed(linux)/fail(windows) depending on OS.+ // If it failed, manually remove the old file and then rename.+ // Because this is a non-atomic operation, its best to do this when holding a fs lock+ if (destFile.exists()) {+ checkedDelete(destFile);+ }+ checkedRename(destTmpFile, destFile);+ }+ }+ catch (IOException | RecognitionException e) {+ throw new SleighException("Error compiling %s".formatted(slaSpecFile), e);+ }+ }++ public interface CheckedRunnable<E extends Throwable> {+ void run() throws E;+ }++ private static void checkedDelete(File f) throws IOException {+ if (!f.delete()) {+ throw new IOException("Unable to delete previous file %s".formatted(f));+ }+ }++ private static void checkedRename(File srcFile, File destFile) throws IOException {+ if (!srcFile.renameTo(destFile)) {+ throw new IOException(+ "Failed to rename temp file [%s] to [%s]".formatted(srcFile, destFile));+ }+ }++ /**+ * Offset in the lock file of where to place the lock so it doesn't interfere with+ * reading the contents of the file from processes that don't have the lock (typically only+ * an issue on windows).+ */+ private static final int LOCKFILE_LOCK_OFFSET = 1_000_000;++ /**+ * <pre>+ * Raw: .*(\/|\\)\.\.?(\/|\\)|\.(\/|\\)|\.\.(\/|\\)+ * Parts: .*(\/|\\)\.\.?(\/|\\) - optional text followed by a forward or back slash,+ * followed by one or two literal dots, followed+ * by a forward or back slash+ * OR+ * \.(\/|\\) - a literal dot followed by a forward or back slash+ * OR+ * \.\.(\/|\\) - two literal dots followed by a forward or back slash+ * </pre>+ */+ private static final Pattern RELATIVE_PATHS_PATTERN =+ Pattern.compile(".*(\\/|\\\\)\\.\\.?(\\/|\\\\)|\\.(\\/|\\\\)|\\.\\.(\\/|\\\\)");++ private static String discardRelativePath(String str) {+ return RELATIVE_PATHS_PATTERN.matcher(str).replaceFirst("");+ }++ private static void writeLockerInfo(RandomAccessFile raf) throws IOException {+ long mypid = ProcessHandle.current().pid();+ raf.setLength(0);+ raf.write("%d\n".formatted(mypid).getBytes(StandardCharsets.UTF_8));+ }++ private static long tryReadLockerInfo(RandomAccessFile raf) {+ try {+ byte[] buffer = new byte[64];+ raf.seek(0);+ int bytesRead = raf.read(buffer);+ if (bytesRead > 0 && buffer[bytesRead - 1] == '\n') {+ // low-tech verification of the data by checking for a trailing \n+ String s = new String(buffer, 0, bytesRead - 1, StandardCharsets.UTF_8);+ long lockersPid = Long.parseLong(s);+ return lockersPid;+ }+ }+ catch (IOException | NumberFormatException e) {+ // fall thru+ }+ return -1;+ }++ private static ResourceFile getSleighArgsForSlaSpec(ResourceFile slaSpecFile) {+ ResourceFile languageModule = Application.getModuleContainingResourceFile(slaSpecFile);+ if (languageModule == null) {+ return null;+ }+ return new ResourceFile(languageModule,+ SystemUtilities.isInReleaseMode() ? "data/sleighArgs.txt" : "build/tmp/sleighArgs.txt");+ }++ private static ResourceFile findFile(ResourceFile parentDir, String fileNameOrRelativePath,+ String extension) {+ ResourceFile file = new ResourceFile(parentDir, fileNameOrRelativePath).getCanonicalFile();+ if (file.exists()) {+ return file;+ }++ String fileName = FilenameUtils.getName(fileNameOrRelativePath);+ List<ResourceFile> files = findFiles(fileName, extension);+ if (files.size() == 1) {+ return files.get(0);+ }++ String relativePath = discardRelativePath(fileNameOrRelativePath);+ for (ResourceFile resourceFile : files) {+ if (file.getAbsolutePath().endsWith(relativePath)) {+ return resourceFile;+ }+ }+ return null;+ }++ private static List<ResourceFile> findFiles(String fileName, String extension) {+ List<ResourceFile> matches = new ArrayList<ResourceFile>();+ List<ResourceFile> files = Application.findFilesByExtensionInApplication(extension);+ for (ResourceFile resourceFile : files) {+ if (resourceFile.getName().equals(fileName)) {+ matches.add(resourceFile);+ }+ }+ return matches;+ }++}diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProvider.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProvider.javaindex b14f20e98c..2c8d2287ac 100644--- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProvider.java+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProvider.java@@ -4,9 +4,9 @@* Licensed under the Apache License, Version 2.0 (the "License");* you may not use this file except in compliance with the License.* You may obtain a copy of the License at- *+ ** http://www.apache.org/licenses/LICENSE-2.0- *+ ** Unless required by applicable law or agreed to in writing, software* distributed under the License is distributed on an "AS IS" BASIS,* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.@@ -17,19 +17,18 @@ package ghidra.app.plugin.processors.sleigh;import static utilities.util.FileUtilities.*;-import java.io.FileNotFoundException;import java.io.IOException;+import java.time.Duration;import java.util.*;-import java.util.regex.Pattern;import org.xml.sax.*;import generic.jar.ResourceFile;import ghidra.framework.Application;import ghidra.program.model.lang.*;-import ghidra.program.model.pcode.DecoderException;import ghidra.util.Msg;import ghidra.util.SystemUtilities;+import ghidra.util.task.TaskMonitor;import ghidra.util.xml.SpecXmlUtils;import ghidra.xml.*;import utilities.util.FileResolutionResult;@@ -39,32 +38,11 @@ import utilities.util.FileResolutionResult;* specifications*/public class SleighLanguageProvider implements LanguageProvider {-/**- * <pre>- * Raw: .*(\/|\\)\.\.?(\/|\\)|\.(\/|\\)|\.\.(\/|\\)- * Parts: .*(\/|\\)\.\.?(\/|\\) - optional text followed by a forward or back slash,- * followed by one or two literal dots, followed- * by a forward or back slash- * OR- * \.(\/|\\) - a literal dot followed by a forward or back slash- * OR- * \.\.(\/|\\) - two literal dots followed by a forward or back slash- * </pre>+ * Returns a singleton instance of a {@link SleighLanguageProvider}.+ *+ * @return singleton {@link SleighLanguageProvider}*/- private static final Pattern RELATIVE_PATHS_PATTERN =- Pattern.compile(".*(\\/|\\\\)\\.\\.?(\\/|\\\\)|\\.(\\/|\\\\)|\\.\\.(\\/|\\\\)");-- private final LinkedHashMap<LanguageID, SleighLanguage> languages =- new LinkedHashMap<LanguageID, SleighLanguage>();- private final LinkedHashMap<LanguageID, SleighLanguageDescription> descriptions =- new LinkedHashMap<LanguageID, SleighLanguageDescription>();- private int failureCount = 0;-- public final static String LANGUAGE_DIR_NAME = "languages";-- private static SleighLanguageProvider instance; // sleigh language provider instance (singleton)-public static synchronized SleighLanguageProvider getSleighLanguageProvider() {if (instance == null) {instance = new SleighLanguageProvider();@@ -72,6 +50,19 @@ public class SleighLanguageProvider implements LanguageProvider {return instance;}+ private static SleighLanguageProvider instance; // sleigh language provider instance (singleton)++ /**+ * Property that can be set as a jvm startup option to control the sla lock timeout duration.+ * <p>+ * See {@link #LANGUAGE_LOCK_TIMEOUT}.+ */+ public static final String LANGUAGE_LOCK_TIMEOUT_PROPNAME =+ "ghidra.app.plugin.processors.sleigh.SleighLanguageProvider.LANGUAGE_LOCK_TIMEOUT_MS";++ private final Map<LanguageID, LanguageRec> languages = new LinkedHashMap<>(); // preserve load order+ private int failureCount = 0;+/*** Construct sleigh language provider (singleton use)*/@@ -81,7 +72,7 @@ public class SleighLanguageProvider implements LanguageProvider {}catch (Exception e) {Msg.error(SleighLanguageProvider.class,- "Sleigh language provider initiailization failed", e);+ "Sleigh language provider initialization failed", e);}}@@ -109,8 +100,14 @@ public class SleighLanguageProvider implements LanguageProvider {}catch (SleighException e) {++failureCount;- Msg.showError(this, null, "Problem loading " + file.getName(),- "Validation error: " + e.getMessage(), e);+ if (e instanceof SleighFileException) {+ // no need for stack trace+ Msg.showError(this, null, "Problem loading " + file.getName(), e.getMessage());+ }+ else {+ Msg.showError(this, null, "Problem loading " + file.getName(),+ "Validation error: " + e.getMessage(), e);+ }}}@@ -121,83 +118,52 @@ public class SleighLanguageProvider implements LanguageProvider {@Overridepublic boolean isLanguageLoaded(LanguageID languageId) {- return languages.get(languageId) != null;+ return languages.getOrDefault(languageId, LanguageRec.NOT_FOUND).lang != null;}@Override- public Language getLanguage(LanguageID languageId) {- SleighLanguageDescription description = descriptions.get(languageId);- SleighLanguage lang = languages.get(languageId);- if (lang == null && description != null) {- try {- lang = new SleighLanguage(description);- languages.put(languageId, lang);- }- catch (SleighException e) {- Msg.showError(this, null, "Error",- "Can't read language spec " + description.getSlaFile().getAbsolutePath(), e);- throw e;- }- catch (DecoderException e) {- Msg.showError(this, null, "Error",- "Can't read language spec " + description.getSlaFile().getAbsolutePath(), e);- throw new SleighException("Format violation in the .sla file", e);- }- catch (FileNotFoundException e) {- Msg.showError(this, null, "Error",- "Can't read language spec " + description.getSlaFile().getAbsolutePath(), e);- throw new SleighException(- "File not found - language probably did not compile properly", e);- }- catch (SAXException e) {- Msg.showError(this, null, "Error",- "Can't read language spec " + description.getSlaFile().getAbsolutePath(), e);- throw new SleighException(- "SAXException - language probably did not compile properly", e);- }- catch (IOException e) {- Msg.showError(this, null, "Error",- "Can't read language spec " + description.getSlaFile().getAbsolutePath(), e);- throw new SleighException(- "IOException - language probably did not compile properly", e);+ public SleighLanguage getLanguage(LanguageID languageId, TaskMonitor monitor)+ throws LanguageNotFoundException {+ LanguageRec langRec = languages.getOrDefault(languageId, LanguageRec.NOT_FOUND);+ if (langRec != LanguageRec.NOT_FOUND && langRec.lang == null) {+ if (langRec.isRepeatFailedLangFile()) {+ throw new LanguageNotFoundException(languageId, langRec.th);}+ langRec.loadLanguage(monitor);}- return lang;+ return langRec.lang;}- void unloadLanguage(LanguageID languageID) {- if (languages.containsKey(languageID)) {- languages.put(languageID, null);++ /**+ * Returns the {@link SleighLanguageDescription language description} of the specified language+ *+ * @param languageId {@link LanguageID}+ * @return {@link SleighLanguageDescription}+ */+ public SleighLanguageDescription getLanguageDescription(LanguageID languageId) {+ return languages.getOrDefault(languageId, LanguageRec.NOT_FOUND).langDesc;+ }++ void unloadLanguage(LanguageID languageId) {+ LanguageRec langRec = languages.get(languageId);+ if (langRec != null) {+ langRec.unloadLanguage();}}@Overridepublic LanguageDescription[] getLanguageDescriptions() {- LanguageDescription[] d = new LanguageDescription[descriptions.size()];- descriptions.values().toArray(d);- return d;+ return languages.values()+ .stream()+ .map(langRec -> langRec.langDesc)+ .toArray(LanguageDescription[]::new);}- private void createLanguageDescriptions(final ResourceFile specFile)+ private void createLanguageDescriptions(ResourceFile specFile)throws SAXException, IOException {- ErrorHandler errHandler = new ErrorHandler() {- @Override- public void error(SAXParseException exception) throws SAXException {- Msg.error(SleighLanguageProvider.this, "Error parsing " + specFile, exception);- }-- @Override- public void fatalError(SAXParseException exception) throws SAXException {- Msg.error(SleighLanguageProvider.this, "Fatal error parsing " + specFile,- exception);- }-- @Override- public void warning(SAXParseException exception) throws SAXException {- Msg.warn(SleighLanguageProvider.this, "Warning parsing " + specFile, exception);- }- };- XmlPullParser parser = XmlPullParserFactory.create(specFile, errHandler, false);+ XmlPullParser parser =+ XmlPullParserFactory.create(specFile, loggingErrorHandler(specFile), false);try {read(parser, specFile.getParentFile(), specFile.getName());}@@ -294,17 +260,12 @@ public class SleighLanguageProvider implements LanguageProvider {}while ((compiler = parser.softStart("compiler")) != null) {String compilerID = compiler.getAttribute("id");- final CompilerSpecID compilerSpecID = new CompilerSpecID(compilerID);- final String compilerSpecName = compiler.getAttribute("name");- final String compilerSpecFilename = compiler.getAttribute("spec");- final ResourceFile compilerSpecFile =- findFile(parentDirectory, compilerSpecFilename, ".cspec");- FileResolutionResult result = existsAndIsCaseDependent(compilerSpecFile);- if (!result.isOk()) {- throw new SleighException("cspec file " + compilerSpecFile +- " is not properly case dependent: " + result.getMessage());- }- final SleighCompilerSpecDescription sleighCompilerSpecDescription =+ CompilerSpecID compilerSpecID = new CompilerSpecID(compilerID);+ String compilerSpecName = compiler.getAttribute("name");+ String compilerSpecFilename = compiler.getAttribute("spec");+ ResourceFile compilerSpecFile = SleighLanguageFile+ .getLanguageResourceFile(parentDirectory, compilerSpecFilename, ".cspec");+ SleighCompilerSpecDescription sleighCompilerSpecDescription =new SleighCompilerSpecDescription(compilerSpecID, compilerSpecName,compilerSpecFile);compilerSpecs.add(sleighCompilerSpecDescription);@@ -343,58 +304,26 @@ public class SleighLanguageProvider implements LanguageProvider {" is not properly case dependent: " + result.getMessage());}description.setDefsFile(defsFile);- final ResourceFile specFile = findFile(parentDirectory, pspec, ".pspec");- result = existsAndIsCaseDependent(specFile);- if (!result.isOk()) {- throw new SleighException("pspec file " + specFile +- " is not properly case dependent: " + result.getMessage());- }- description.setSpecFile(specFile);-- ResourceFile slaFile;- try {- slaFile = findFile(parentDirectory, slafilename, ".slaspec");- result = existsAndIsCaseDependent(slaFile);- if (!result.isOk()) {- throw new SleighException("sla file " + slaFile +- " is not properly case dependent: " + result.getMessage());- }- description.setSlaFile(slaFile);- }- catch (SleighException e) {- int index = slafilename.lastIndexOf('.');- String slabase = slafilename.substring(0, index);- String slaspecfilename = slabase + ".slaspec";+ ResourceFile specFile =+ SleighLanguageFile.getLanguageResourceFile(parentDirectory, pspec, ".pspec");+ description.setSpecFile(specFile);- ResourceFile slaspecFile = findFile(parentDirectory, slaspecfilename, ".slaspec");- result = existsAndIsCaseDependent(slaspecFile);- if (!result.isOk()) {- throw new SleighException("sla file source " + slaspecFile +- " is not properly case dependent: " + result.getMessage());- }-- slaFile = new ResourceFile(slaspecFile.getParentFile(), slafilename);- description.setSlaFile(slaFile);- }+ SleighLanguageFile langFile =+ SleighLanguageFile.fromSlaFilename(parentDirectory, slafilename);+ description.setLanguageFile(langFile);try {if (manualindexfile != null) {- ResourceFile manualIndexFile =- findFile(parentDirectory, manualindexfile, ".idx");- result = existsAndIsCaseDependent(manualIndexFile);- if (result.isOk()) {- description.setManualIndexFile(manualIndexFile);- }- else {- throw new SleighException(result.getMessage());- }+ ResourceFile manualIndexFile = SleighLanguageFile+ .getLanguageResourceFile(parentDirectory, manualindexfile, ".idx");+ description.setManualIndexFile(manualIndexFile);}}catch (SleighException ex) { // Error with the manual shouldn't prevent language from loadingMsg.error(this, ex.getMessage());}- if (descriptions.put(id, description) != null) {+ if (languages.put(id, new LanguageRec(description)) != null) {Msg.showError(this, null, "Duplicate Sleigh Language ID","Language " + id + " previously defined: " + defsFile);}@@ -402,48 +331,155 @@ public class SleighLanguageProvider implements LanguageProvider {parser.end(start);}- private ResourceFile findFile(ResourceFile parentDir, String fileNameOrRelativePath,- String extension) throws SleighException {- ResourceFile file = new ResourceFile(parentDir, fileNameOrRelativePath);- if (file.exists()) {- return file;+ //---------------------------------------------------------------------------------------------+ /**+ * Timeout used when trying to acquire the sla file lock. (Default: 60 seconds)+ * <p>+ * The sla lock is acquired and held during .sla file checking and writing (compiling).+ * Currently parsing the sla xml is done after releasing the lock.+ * <p>+ * If a Ghidra process is trying to fetch a sleigh language, which requires acquiring the lock+ * on a sla file, and it times out before succeeding, the caller will get a+ * {@link LanguageNotFoundException} exception.+ * <p>+ * This timeout should be long enough to allow the process that has the lock to finish+ * compiling and writing the slaspec so that the waiting process does not give up too quickly+ * and give an error to the user.+ *+ * See {@link #LANGUAGE_LOCK_TIMEOUT_PROPNAME}.+ */+ public static final Duration LANGUAGE_LOCK_TIMEOUT = getLanguageLockTimeout();++ private static final int DEFAULT_LOCK_TIMEOUT_SECS = 60;++ private static Duration getLanguageLockTimeout() {+ String langLockTimeoutOverride = System.getProperty(LANGUAGE_LOCK_TIMEOUT_PROPNAME);+ if (langLockTimeoutOverride != null) {+ try {+ return Duration.ofMillis(Long.parseLong(langLockTimeoutOverride));+ }+ catch (NumberFormatException e) {+ // fallthru+ }}- String fileName = getFileNameFromPath(fileNameOrRelativePath);- List<ResourceFile> files = findFiles(fileName, extension);- if (files.size() == 1) {- return files.get(0);+ return Duration.ofSeconds(DEFAULT_LOCK_TIMEOUT_SECS);+ }++ private static class LanguageRec {+ static final LanguageRec NOT_FOUND = new LanguageRec(null);++ SleighLanguage lang;+ SleighLanguageDescription langDesc;+ Long badSlaspecTS;+ Throwable th;++ LanguageRec(SleighLanguageDescription langDesc) {+ this.langDesc = langDesc;}- String relativePath = discardRelativePath(fileNameOrRelativePath);- for (ResourceFile resourceFile : files) {- if (file.getAbsolutePath().endsWith(relativePath)) {- return resourceFile;+ void markLangFileFailed(Throwable e) {+ ResourceFile slaSpecFile = langDesc.getLanguageFile().getSlaSpecFile();+ badSlaspecTS = slaSpecFile.lastModified();+ th = e;+ }++ boolean isRepeatFailedLangFile() {+ if (th instanceof SleighFileLockException) {+ // allow user to retry getting langauge because the exception would have had a built-in delay+ return false;+ }+ // Prevents trying to load the same language over and over again, but still allows the+ // user/developer to fix a slaspec file (and change its timestamp) and try to load it+ // again without having to restart ghidra. Does not consider timestamps of sinc files.+ if (badSlaspecTS != null) {+ ResourceFile slaSpecFile = langDesc.getLanguageFile().getSlaSpecFile();+ long slaSpecTS = slaSpecFile.lastModified();+ if (slaSpecTS != badSlaspecTS) {+ badSlaspecTS = null;+ }+ }+ return badSlaspecTS != null;+ }++ /**+ * Loads the language.+ *+ * @param monitor {@link TaskMonitor}. See+ * {@link SleighLanguage#SleighLanguage(SleighLanguageDescription, TaskMonitor)} for notes+ * about how the TaskMonitor's settings are modified during loading.+ * @throws LanguageNotFoundException if error reading language info+ */+ void loadLanguage(TaskMonitor monitor) throws LanguageNotFoundException {+ try {+ lang = new SleighLanguage(langDesc, monitor);+ badSlaspecTS = null;+ }+ catch (SleighException e) {+ markLangFileFailed(e);+ if (!(e instanceof SleighFileException)) {+ // don't force showing error if its just a missing file because it will be displayed elsewhere+ Msg.showError(this, null, "Error",+ "Failed to read language %s".formatted(langDesc.getLanguageFile()), e);+ }+ throw new LanguageNotFoundException(langDesc.getLanguageID(), e);}}- ResourceFile missingFile = new ResourceFile(parentDir, fileNameOrRelativePath);- throw new SleighException("Missing sleigh file: " + missingFile.getAbsolutePath());- }- private String discardRelativePath(String str) {- return RELATIVE_PATHS_PATTERN.matcher(str).replaceFirst("");+ void unloadLanguage() {+ lang = null;+ badSlaspecTS = null;+ th = null;+ }}- private List<ResourceFile> findFiles(String fileName, String extension) {- List<ResourceFile> matches = new ArrayList<ResourceFile>();- List<ResourceFile> files = Application.findFilesByExtensionInApplication(extension);- for (ResourceFile resourceFile : files) {- if (resourceFile.getName().equals(fileName)) {- matches.add(resourceFile);+ /**+ * Creates a SAX ErrorHandler that re-throws the exceptions, ignores the warning+ *+ * @return new {@link ErrorHandler}+ */+ static ErrorHandler throwingErrorHandler() {+ return new ErrorHandler() {+ @Override+ public void warning(SAXParseException exception) throws SAXException {+ // ignore+ }++ @Override+ public void fatalError(SAXParseException exception) throws SAXException {+ throw exception;}- }- return matches;++ @Override+ public void error(SAXParseException exception) throws SAXException {+ throw exception;+ }+ };}- private String getFileNameFromPath(String fileNameOrRelativePath) {- int lastIndexOf = fileNameOrRelativePath.lastIndexOf("/");- if (lastIndexOf < 0) {- return fileNameOrRelativePath;- }- return fileNameOrRelativePath.substring(lastIndexOf + 1);+ /**+ * Creates a SAX ErrorHandler that logs the exceptions+ *+ * @param specFile the input file+ * @return new {@link ErrorHandler}+ */+ static ErrorHandler loggingErrorHandler(ResourceFile specFile) {+ return new ErrorHandler() {+ @Override+ public void error(SAXParseException exception) throws SAXException {+ Msg.error(SleighLanguageProvider.class, "Error parsing " + specFile, exception);+ }++ @Override+ public void fatalError(SAXParseException exception) throws SAXException {+ Msg.error(SleighLanguageProvider.class, "Fatal error parsing " + specFile,+ exception);+ }++ @Override+ public void warning(SAXParseException exception) throws SAXException {+ Msg.warn(SleighLanguageProvider.class, "Warning parsing " + specFile, exception);+ }+ };}+}diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcode/utils/SlaFormat.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcode/utils/SlaFormat.javaindex 742341daa6..0e3a299292 100644--- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcode/utils/SlaFormat.java+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcode/utils/SlaFormat.java@@ -200,18 +200,26 @@ public class SlaFormat {* @throws IOException for any errors reading from the stream*/public static boolean isSlaFormat(InputStream stream) throws IOException {+ return getSlaFormat(stream) == FORMAT_VERSION;+ }++ /**+ * Returns the format version number of the specified binary .sla file.+ *+ * @param stream {@link InputStream}+ * @return sla version number, -1 if invalid header+ * @throws IOException if error reading+ */+ public static int getSlaFormat(InputStream stream) throws IOException {byte[] header = new byte[4];int readLen = stream.read(header);if (readLen < 4) {- return false;+ return -1;}if (header[0] != 's' || header[1] != 'l' || header[2] != 'a') {- return false;- }- if (header[3] != FORMAT_VERSION) {- return false;+ return -1;}- return true;+ return Byte.toUnsignedInt(header[3]);}/**diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompile.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompile.javaindex 9b63f8172b..295fbbc141 100644--- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompile.java+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompile.java@@ -1784,6 +1784,22 @@ public class SleighCompile extends SleighBase {return 0;}+ public void setOptions(SleighCompileOptions options) {+ Set<Entry<String, String>> entrySet = options.preprocs.entrySet();+ for (Entry<String, String> entry : entrySet) {+ setPreprocValue(entry.getKey(), entry.getValue());+ }+ setUnnecessaryPcodeWarning(options.unnecessaryPcodeWarning);+ setLenientConflict(options.lenientConflict);+ setLocalCollisionWarning(options.allCollisionWarning);+ setAllNopWarning(options.allNopWarning);+ setDeadTempWarning(options.deadTempWarning);+ setUnusedFieldWarning(options.unusedFieldWarning);+ setEnforceLocalKeyWord(options.enforceLocalKeyWord);+ setInsensitiveDuplicateError(!options.caseSensitiveRegisterNames);+ setDebugOutput(options.debugOutput);+ }+public void setAllOptions(Map<String, String> preprocs, boolean unnecessaryPcodeWarning,boolean lenientConflict, boolean allCollisionWarning, boolean allNopWarning,boolean deadTempWarning, boolean unusedFieldWarning, boolean enforceLocalKeyWord,diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileLauncher.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileLauncher.javaindex 14223ccc3d..6b034433ad 100644--- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileLauncher.java+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileLauncher.java@@ -16,18 +16,16 @@package ghidra.pcodeCPort.slgh_compile;import java.io.*;-import java.util.*;+import java.util.TreeSet;import org.antlr.runtime.RecognitionException;import org.jdom2.JDOMException;-import generic.jar.ResourceFile;import ghidra.GhidraApplicationLayout;import ghidra.GhidraLaunchable;+import ghidra.app.plugin.processors.sleigh.SleighException;import ghidra.framework.Application;import ghidra.framework.ApplicationConfiguration;-import ghidra.util.Msg;-import ghidra.util.SystemUtilities;/*** <code>SleighCompileLauncher</code> Sleigh compiler launch provider@@ -59,250 +57,68 @@ public class SleighCompileLauncher implements GhidraLaunchable {* @throws RecognitionException for parse errors*/public static int runMain(String[] args) throws IOException, RecognitionException {- int retval;- String filein = null;- String fileout = null;- Map<String, String> preprocs = new HashMap<>();-- SleighCompile.yydebug = false;- boolean allMode = false;-- if (args.length < 1) {- // @formatter:off- Msg.info(SleighCompile.class, "Usage: sleigh [options...] [<infile.slaspec> [<outfile.sla>] | -a <directory-path>]");- Msg.info(SleighCompile.class, " sleigh [options...] <infile.slaspec> [<outfile.sla>]");- Msg.info(SleighCompile.class, " <infile.slaspec> source slaspec file to be compiled");- Msg.info(SleighCompile.class, " <outfile.sla> optional output sla file (infile.sla assumed)");- Msg.info(SleighCompile.class, " or");- Msg.info(SleighCompile.class, " sleigh [options...] -a <directory-path>");- Msg.info(SleighCompile.class, " <directory-path> directory to have all slaspec files compiled");- Msg.info(SleighCompile.class, " options:");- Msg.info(SleighCompile.class, " -x turns on parser debugging");- Msg.info(SleighCompile.class, " -y write .sla using XML debug format");- Msg.info(SleighCompile.class, " -u print warnings for unnecessary pcode instructions");- Msg.info(SleighCompile.class, " -l report pattern conflicts");- Msg.info(SleighCompile.class, " -n print warnings for all NOP constructors");- Msg.info(SleighCompile.class, " -t print warnings for dead temporaries");- Msg.info(SleighCompile.class, " -e enforce use of 'local' keyword for temporaries");- Msg.info(SleighCompile.class, " -c print warnings for all constructors with colliding operands");- Msg.info(SleighCompile.class, " -f print warnings for unused token fields");- Msg.info(SleighCompile.class, " -s treat register names as case sensitive");- Msg.info(SleighCompile.class, " -DNAME=VALUE defines a preprocessor macro NAME with value VALUE (option may be repeated)");- Msg.info(SleighCompile.class, " -dMODULE defines a preprocessor macro MODULE with a value of its module path (option may be repeated)");- Msg.info(SleighCompile.class, " -i <options-file> inject options from specified file");- // @formatter:on+ if (args.length == 0) {+ SleighCompileOptions.usage();return 2;}-- boolean unnecessaryPcodeWarning = false;- boolean lenientConflict = true;- boolean allCollisionWarning = false;- boolean allNopWarning = false;- boolean deadTempWarning = false;- boolean enforceLocalKeyWord = false;- boolean unusedFieldWarning = false;- boolean caseSensitiveRegisterNames = false;- boolean debugOutput = false;-- int i;- for (i = 0; i < args.length; ++i) {- if (args[i].charAt(0) != '-') {- break;- }- else if (args[i].charAt(1) == 'i') {- // inject options from file specified by next argument- args = injectOptionsFromFile(args, ++i);- if (args == null) {- return 1;- }- }- else if (args[i].charAt(1) == 'D') {- String preproc = args[i].substring(2);- int pos = preproc.indexOf('=');- if (pos == -1) {- Msg.error(SleighCompile.class, "Bad sleigh option: " + args[i]);- return 1;- }- String name = preproc.substring(0, pos);- String value = preproc.substring(pos + 1);- preprocs.put(name, value); // Preprocessor macro definitions- }- else if (args[i].charAt(1) == 'd') {- String moduleName = args[i].substring(2);- ResourceFile module = Application.getModuleRootDir(moduleName);- if (module == null || !module.isDirectory()) {- Msg.error(SleighCompile.class,- "Failed to resolve module reference: " + args[i]);- return 1;- }- Msg.debug(SleighCompile.class,- "Sleigh resolved module: " + moduleName + "=" + module.getAbsolutePath());- preprocs.put(moduleName, module.getAbsolutePath()); // Preprocessor macro definitions- }- else if (args[i].charAt(1) == 'u') {- unnecessaryPcodeWarning = true;- }- else if (args[i].charAt(1) == 't') {- deadTempWarning = true;- }- else if (args[i].charAt(1) == 'e') {- enforceLocalKeyWord = true;- }- else if (args[i].charAt(1) == 'f') {- unusedFieldWarning = true;- }- else if (args[i].charAt(1) == 'l') {- lenientConflict = false;- }- else if (args[i].charAt(1) == 'c') {- allCollisionWarning = true;- }- else if (args[i].charAt(1) == 'n') {- allNopWarning = true;- }- else if (args[i].charAt(1) == 'a') {- allMode = true;- }- else if (args[i].charAt(1) == 's') {- caseSensitiveRegisterNames = true;- }- else if (args[i].charAt(1) == 'y') {- debugOutput = true;- }- else if (args[i].charAt(1) == 'x') {- SleighCompile.yydebug = true; // Debug option- }- else {- Msg.error(SleighCompile.class, "Unknown option: " + args[i]);- return 1;- }+ SleighCompileOptions options;+ try {+ options = SleighCompileOptions.parse(args);}-- if (i < args.length - 2) {- Msg.error(SleighCompile.class, "Too many parameters");- return 1;- }-- if (allMode) {- if (i == args.length) {- Msg.error(SleighCompile.class, "Missing input directory path");- return 1;- }- String directory = args[i];- File dir = new File(directory);- if (!dir.exists() || !dir.isDirectory()) {- Msg.error(SleighCompile.class, directory + " is not a directory");- return 1;- }- TreeSet<String> failures = new TreeSet<>();- int totalFailures = 0;- int totalSuccesses = 0;- DirectoryVisitor visitor = new DirectoryVisitor(dir, SLASPEC_FILTER);- for (File input : visitor) {- System.out.println("Compiling " + input + ":");- SleighCompile compiler = new SleighCompile();- compiler.setAllOptions(preprocs, unnecessaryPcodeWarning, lenientConflict,- allCollisionWarning, allNopWarning, deadTempWarning, unusedFieldWarning,- enforceLocalKeyWord, caseSensitiveRegisterNames,- debugOutput);-- String outname = input.getName().replace(".slaspec", ".sla");- File output = new File(input.getParent(), outname);- retval =- compiler.run_compilation(input.getAbsolutePath(), output.getAbsolutePath());- System.out.println();- if (retval != 0) {- ++totalFailures;- failures.add(input.getAbsolutePath());- }- else {- ++totalSuccesses;- }- }- System.out.println(totalSuccesses + " languages successfully compiled");- if (totalFailures != 0) {- for (String path : failures) {- System.out.println(path + " failed to compile");- }- System.out.println(totalFailures + " languages total failed to compile");- }- return -totalFailures;- }-- // single file compile- SleighCompile compiler = new SleighCompile();- compiler.setAllOptions(preprocs, unnecessaryPcodeWarning, lenientConflict,- allCollisionWarning, allNopWarning, deadTempWarning, unusedFieldWarning,- enforceLocalKeyWord, caseSensitiveRegisterNames, debugOutput);- if (i == args.length) {- Msg.error(SleighCompile.class, "Missing input file name");+ catch (SleighException e) {return 1;}+ return launchCompile(options);+ }- filein = args[i];- if (i < args.length - 1) {- fileout = args[i + 1];+ public static int launchCompile(SleighCompileOptions options)+ throws IOException, RecognitionException {+ if (options.allMode) {+ return compileAll(options);}-- String baseName = filein;- if (filein.toLowerCase().endsWith(FILE_IN_DEFAULT_EXT)) {- baseName = filein.substring(0, filein.length() - FILE_IN_DEFAULT_EXT.length());+ else {+ return compileOne(options);}- filein = baseName + FILE_IN_DEFAULT_EXT;+ }- String baseOutName = fileout;- if (fileout == null) {- baseOutName = baseName;- }- else if (fileout.toLowerCase().endsWith(FILE_OUT_DEFAULT_EXT)) {- baseOutName = fileout.substring(0, fileout.length() - FILE_OUT_DEFAULT_EXT.length());- }- fileout = baseOutName + FILE_OUT_DEFAULT_EXT;+ public static int compileOne(SleighCompileOptions options)+ throws IOException, RecognitionException {+ SleighCompile compiler = new SleighCompile();+ compiler.setOptions(options);- return compiler.run_compilation(filein, fileout);+ return compiler.run_compilation(options.inputFile.getPath(), options.outputFile.getPath());}- private static String[] injectOptionsFromFile(String[] args, int index) {- if (index >= args.length) {- Msg.error(SleighCompile.class, "Missing options input file name");- return null;- }-- File optionsFile = new File(args[index]);- if (!optionsFile.isFile()) {- Msg.error(SleighCompile.class,- "Options file not found: " + optionsFile.getAbsolutePath());- if (SystemUtilities.isInDevelopmentMode()) {- Msg.error(SleighCompile.class,- "Eclipse language module must be selected and 'gradle prepdev' prevously run");+ public static int compileAll(SleighCompileOptions options)+ throws IOException, RecognitionException {+ TreeSet<String> failures = new TreeSet<>();+ int totalFailures = 0;+ int totalSuccesses = 0;+ DirectoryVisitor visitor = new DirectoryVisitor(options.allDir, SLASPEC_FILTER);+ for (File input : visitor) {+ System.out.println("Compiling " + input + ":");+ SleighCompile compiler = new SleighCompile();+ compiler.setOptions(options);++ String outname = input.getName().replace(".slaspec", ".sla");+ File output = new File(input.getParentFile(), outname);+ int retval = compiler.run_compilation(input.getPath(), output.getPath());+ System.out.println();+ if (retval != 0) {+ ++totalFailures;+ failures.add(input.getPath());}- return null;- }- ArrayList<String> list = new ArrayList<>();- for (int i = 0; i <= index; i++) {- list.add(args[i]);- }-- try (BufferedReader r = new BufferedReader(new FileReader(optionsFile))) {- String option = r.readLine();- while (option != null) {- option = option.trim();- if (option.length() != 0 && !option.startsWith("#")) {- list.add(option);- }- option = r.readLine();+ else {+ ++totalSuccesses;}}- catch (IOException e) {- Msg.error(SleighCompile.class,- "Reading options file failed (" + optionsFile.getName() + "): " + e.getMessage());- return null;- }-- for (int i = index + 1; i < args.length; i++) {- list.add(args[i]);+ System.out.println(totalSuccesses + " languages successfully compiled");+ if (totalFailures != 0) {+ for (String path : failures) {+ System.out.println(path + " failed to compile");+ }+ System.out.println(totalFailures + " languages total failed to compile");}- return list.toArray(new String[list.size()]);+ return -totalFailures;}-}diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptions.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptions.javanew file mode 100644index 0000000000..78543ca6b2--- /dev/null+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptions.java@@ -0,0 +1,283 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.pcodeCPort.slgh_compile;++import java.io.File;+import java.io.IOException;+import java.util.*;++import org.apache.commons.io.FilenameUtils;++import generic.jar.ResourceFile;+import ghidra.app.plugin.processors.sleigh.SleighException;+import ghidra.app.plugin.processors.sleigh.SleighLanguageFile;+import ghidra.framework.Application;+import ghidra.util.Msg;+import ghidra.util.SystemUtilities;+import utilities.util.FileUtilities;++/**+ * Represents the options the sleigh compiler uses+ */+public class SleighCompileOptions {+ public File inputFile;+ public File outputFile;+ public boolean allMode = false;+ public File allDir;++ public Map<String, String> preprocs = new HashMap<>();+ public boolean unnecessaryPcodeWarning = false;+ public boolean lenientConflict = true;+ public boolean allCollisionWarning = false;+ public boolean allNopWarning = false;+ public boolean deadTempWarning = false;+ public boolean enforceLocalKeyWord = false;+ public boolean unusedFieldWarning = false;+ public boolean caseSensitiveRegisterNames = false;+ public boolean debugOutput = false;++ public static void usage() {+ // @formatter:off+ Msg.info(SleighCompileOptions.class, "Usage: sleigh [options...] [<infile.slaspec> [<outfile.sla>] | -a <directory-path>]");+ Msg.info(SleighCompileOptions.class, " sleigh [options...] <infile.slaspec> [<outfile.sla>]");+ Msg.info(SleighCompileOptions.class, " <infile.slaspec> source slaspec file to be compiled");+ Msg.info(SleighCompileOptions.class, " <outfile.sla> optional output sla file (infile.sla assumed)");+ Msg.info(SleighCompileOptions.class, " or");+ Msg.info(SleighCompileOptions.class, " sleigh [options...] -a <directory-path>");+ Msg.info(SleighCompileOptions.class, " <directory-path> directory to have all slaspec files compiled");+ Msg.info(SleighCompileOptions.class, " options:");+ Msg.info(SleighCompileOptions.class, " -x turns on parser debugging");+ Msg.info(SleighCompileOptions.class, " -y write .sla using XML debug format");+ Msg.info(SleighCompileOptions.class, " -u print warnings for unnecessary pcode instructions");+ Msg.info(SleighCompileOptions.class, " -l report pattern conflicts");+ Msg.info(SleighCompileOptions.class, " -n print warnings for all NOP constructors");+ Msg.info(SleighCompileOptions.class, " -t print warnings for dead temporaries");+ Msg.info(SleighCompileOptions.class, " -e enforce use of 'local' keyword for temporaries");+ Msg.info(SleighCompileOptions.class, " -c print warnings for all constructors with colliding operands");+ Msg.info(SleighCompileOptions.class, " -f print warnings for unused token fields");+ Msg.info(SleighCompileOptions.class, " -s treat register names as case sensitive");+ Msg.info(SleighCompileOptions.class, " -DNAME=VALUE defines a preprocessor macro NAME with value VALUE (option may be repeated)");+ Msg.info(SleighCompileOptions.class, " -dMODULE defines a preprocessor macro MODULE with a value of its module path (option may be repeated)");+ Msg.info(SleighCompileOptions.class, " -i <options-file> inject options from specified file");+ // @formatter:on+ }++ /**+ * Evaluates an array of string arguments and saves the values into a {@link SleighCompileOptions}+ * instance.+ *+ * @param args array of arg strings+ * @return new {@link SleighCompileOptions} instance+ * @throws SleighException if error in an argument+ */+ public static SleighCompileOptions parse(String[] args) throws SleighException {++ SleighCompileOptions results = new SleighCompileOptions();++ Deque<String> argList = new ArrayDeque<>(List.of(args));+ while (!argList.isEmpty()) {+ if (!argList.peekFirst().startsWith("-")) {+ break;+ }+ String arg = argList.removeFirst();+ if (arg.isBlank()) {+ continue;+ }+ results.processArg(arg, argList);+ }++ if (!results.allMode) {+ // Process trailing source and destination filename parameters+ if (argList.isEmpty()) {+ Msg.error(SleighCompileOptions.class, "Missing input file name");+ throw new SleighException("Missing input file name");+ }++ results.inputFile = new File(argList.removeFirst()).getAbsoluteFile();+ results.outputFile =+ !argList.isEmpty() ? new File(argList.removeFirst()).getAbsoluteFile() : null;++ String baseName = results.inputFile.getName();+ if (baseName.toLowerCase().endsWith(SleighLanguageFile.SLASPEC_EXT)) {+ baseName = FilenameUtils.getBaseName(baseName);+ }+ else {+ results.inputFile = new File(results.inputFile.getParentFile(),+ results.inputFile.getName() + SleighLanguageFile.SLASPEC_EXT);+ }++ if (results.outputFile == null) {+ results.outputFile = new File(results.inputFile.getParentFile(),+ baseName + SleighLanguageFile.SLA_EXT);+ }+ else if (!results.outputFile.getName()+ .toLowerCase()+ .endsWith(SleighLanguageFile.SLA_EXT)) {+ results.outputFile = new File(results.outputFile.getParentFile(),+ results.outputFile.getName() + SleighLanguageFile.SLA_EXT);+ }+ }++ if (!argList.isEmpty()) {+ Msg.error(SleighCompileOptions.class, "Too many parameters: " + argList.toString());+ throw new SleighException("Too many parameters: " + argList.toString());+ }+++ return results;+ }++ /**+ * Evaluates the arguments in a sleighArgs.txt file and returns a {@link SleighCompileOptions}+ *+ * @param argsFile file containing an argument per line+ * @return {@link SleighCompileOptions}+ */+ public static SleighCompileOptions fromFile(File argsFile) {+ SleighCompileOptions results = new SleighCompileOptions();++ Deque<String> argList = new ArrayDeque<>(getOptionsFromFile(argsFile));+ while (!argList.isEmpty()) {+ String arg = argList.removeFirst();+ if (arg.isBlank()) {+ continue;+ }+ if (!arg.startsWith("-")) {+ break;+ }+ results.processArg(arg, argList);+ }+ return results;+ }++ public void addPreprocessorMacroDefinition(String name, String value) {+ preprocs.put(name, value); // Preprocessor macro definitions+ }++ private void processArg(String arg, Deque<String> argList) {+ if (arg.length() < 2 || arg.charAt(0) != '-') {+ throw new SleighException("Invalid argument: " + arg);+ }+ switch (arg.charAt(1)) {+ case 'i':+ // inject options from file specified by next argument+ if (argList.isEmpty()) {+ Msg.error(SleighCompileOptions.class, "Missing options input file name");+ throw new SleighException("Missing options input file name");+ }+ String optFilename = argList.removeFirst();+ File optionsFile = new File(optFilename).getAbsoluteFile();+ if (!optionsFile.isFile()) {+ Msg.error(SleighCompileOptions.class, "Options file not found: " + optionsFile);+ if (SystemUtilities.isInDevelopmentMode()) {+ Msg.error(SleighCompileOptions.class,+ "Eclipse language module must be selected and 'gradle prepdev' prevously run");+ }+ throw new SleighException("Options file not found: " + optionsFile);+ }+ Deque<String> newArgList = new ArrayDeque<>(getOptionsFromFile(optionsFile));+ newArgList.addAll(argList);+ argList.clear();+ argList.addAll(newArgList);+ break;+ case 'D':+ String preproc = arg.substring(2);+ int pos = preproc.indexOf('=');+ if (pos == -1) {+ Msg.error(SleighCompileOptions.class, "Bad sleigh option: " + arg);+ throw new SleighException("Bad sleigh option: " + arg);+ }+ String name = preproc.substring(0, pos);+ String value = preproc.substring(pos + 1);+ addPreprocessorMacroDefinition(name, value);+ break;+ case 'd':+ String moduleName = arg.substring(2);+ ResourceFile module = Application.getModuleRootDir(moduleName);+ if (module == null || !module.isDirectory()) {+ Msg.error(SleighCompileOptions.class,+ "Failed to resolve module reference: " + arg);+ throw new SleighException("Failed to resolve module reference: " + arg);+ }+ Msg.debug(SleighCompileOptions.class,+ "Sleigh resolved module: " + moduleName + "=" + module.getAbsolutePath());+ addPreprocessorMacroDefinition(moduleName, module.getAbsolutePath());+ break;+ case 'u':+ unnecessaryPcodeWarning = true;+ break;+ case 't':+ deadTempWarning = true;+ break;+ case 'e':+ enforceLocalKeyWord = true;+ break;+ case 'f':+ unusedFieldWarning = true;+ break;+ case 'l':+ lenientConflict = false;+ break;+ case 'c':+ allCollisionWarning = true;+ break;+ case 'n':+ allNopWarning = true;+ break;+ case 'a':+ if (argList.isEmpty()) {+ Msg.error(SleighCompileOptions.class, "Missing input directory path");+ throw new SleighException("Missing input directory path");+ }+ File dir = new File(argList.removeFirst()).getAbsoluteFile();+ if (!dir.exists() || !dir.isDirectory()) {+ Msg.error(SleighCompileOptions.class, dir + " is not a directory");+ throw new SleighException(dir + " is not a directory");+ }+ allMode = true;+ allDir = dir;+ break;+ case 's':+ caseSensitiveRegisterNames = true;+ break;+ case 'y':+ debugOutput = true;+ break;+ case 'x':+ SleighCompile.yydebug = true; // Debug option+ break;+ default:+ Msg.error(SleighCompileOptions.class, "Unknown option: " + arg);+ throw new SleighException("Unknown option: " + arg);+ }+ }++ private static List<String> getOptionsFromFile(File optionsFile) throws SleighException {+ try {+ return FileUtilities.getLines(optionsFile)+ .stream()+ .map(String::trim)+ .filter(option -> !option.isBlank() && !option.startsWith("#"))+ .toList();+ }+ catch (IOException e) {+ Msg.error(SleighCompileOptions.class,+ "Reading options file failed (" + optionsFile.getName() + "): " + e.getMessage());+ throw new SleighException("Failed reading options file [%s]".formatted(optionsFile), e);+ }+ }++}diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageNotFoundException.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageNotFoundException.javaindex ca94bb00c9..94d9395813 100644--- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageNotFoundException.java+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageNotFoundException.java@@ -1,6 +1,5 @@/* ###* IP: GHIDRA- * REVIEWED: YES** Licensed under the Apache License, Version 2.0 (the "License");* you may not use this file except in compliance with the License.@@ -36,10 +35,21 @@ public class LanguageNotFoundException extends IOException {/*** Language not found- * @param languageID+ *+ * @param languageID {@link LanguageID}*/public LanguageNotFoundException(LanguageID languageID) {- super("Language not found for '" + languageID + "'");+ this(languageID, (Throwable) null);+ }++ /**+ * Language not found because of an exception+ *+ * @param languageID {@link LanguageID}+ * @param cause {@link Throwable} that caused the language to not be found+ */+ public LanguageNotFoundException(LanguageID languageID, Throwable cause) {+ super("Language not found for '" + languageID + "'", cause);}public LanguageNotFoundException(String message) {diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageProvider.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageProvider.javaindex e882e168cf..7477beabc6 100644--- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageProvider.java+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageProvider.java@@ -16,6 +16,7 @@package ghidra.program.model.lang;import ghidra.util.classfinder.ExtensionPoint;+import ghidra.util.task.TaskMonitor;/*** NOTE: ALL LanguageProvider CLASSES MUST END IN "LanguageProvider". If not,@@ -31,9 +32,22 @@ public interface LanguageProvider extends ExtensionPoint {** @param languageId the name of the language to be retrieved* @return the {@link Language} with the given name or null if not found- * @throws RuntimeException if language instantiation error occurs+ * @throws LanguageNotFoundException if language instantiation error*/- Language getLanguage(LanguageID languageId);+ default Language getLanguage(LanguageID languageId) throws LanguageNotFoundException {+ return getLanguage(languageId, TaskMonitor.DUMMY);+ }++ /**+ * Returns the language with the given name or null if no language has that name.+ *+ * @param languageId the name of the language to be retrieved+ * @param monitor {@link TaskMonitor}+ * @return the {@link Language} with the given name or null if not found+ * @throws LanguageNotFoundException if language instantiation error+ */+ Language getLanguage(LanguageID languageId, TaskMonitor monitor)+ throws LanguageNotFoundException;/*** Returns a list of language descriptions provided by this providerdiff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PackedDecode.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PackedDecode.javaindex 13df7a0d20..7516e3a4c5 100644--- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PackedDecode.java+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PackedDecode.java@@ -15,8 +15,7 @@*/package ghidra.program.model.pcode;-import java.io.IOException;-import java.io.InputStream;+import java.io.*;import java.util.ArrayList;import ghidra.program.model.address.AddressFactory;@@ -50,7 +49,7 @@ import ghidra.program.model.address.AddressSpace;* For strings, the integer encoded after the \e type byte, is the actual length of the string. The* string data itself is stored immediately after the length integer using UTF8 format.* */-public class PackedDecode implements Decoder {+public class PackedDecode implements Decoder, Closeable {public static final int HEADER_MASK = 0xc0;public static final int ELEMENT_START = 0x40;@@ -231,6 +230,7 @@ public class PackedDecode implements Decoder {* Close stream cached by the ingestStreamAsNeeded method.* @throws IOException for low-level problems with the stream*/+ @Overridepublic void close() throws IOException {inStream.close();inStream = null;diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/util/DefaultLanguageService.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/util/DefaultLanguageService.javaindex 2a1b28b196..e3cdba3b98 100644--- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/util/DefaultLanguageService.java+++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/util/DefaultLanguageService.java@@ -4,9 +4,9 @@* Licensed under the Apache License, Version 2.0 (the "License");* you may not use this file except in compliance with the License.* You may obtain a copy of the License at- *+ ** http://www.apache.org/licenses/LICENSE-2.0- *+ ** Unless required by applicable law or agreed to in writing, software* distributed under the License is distributed on an "AS IS" BASIS,* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.@@ -16,6 +16,7 @@package ghidra.program.util;import java.util.*;+import java.util.concurrent.atomic.AtomicReference;import org.apache.logging.log4j.LogManager;import org.apache.logging.log4j.Logger;@@ -23,6 +24,7 @@ import org.apache.logging.log4j.Logger;import ghidra.app.plugin.processors.sleigh.SleighLanguageProvider;import ghidra.program.model.lang.*;import ghidra.util.task.TaskBuilder;+import ghidra.util.task.TaskMonitor;/*** Default Language service used gather up all the languages that were found@@ -318,27 +320,54 @@ public class DefaultLanguageService implements LanguageService {this.provider = lp;}- // synchronized to prevent multiple clients from trying to load the language at once- synchronized Language getLanguage() {+ /**+ * Loads a {@link Language} from a {@link LanguageProvider}, using a Task with a modal+ * {@link TaskMonitor}.+ * <p>+ * This is the call path that users will see when using Ghidra and a language is+ * initially loaded from disk.+ *+ * @return {@link Language}+ * @throws LanguageNotFoundException if error initializing Language+ */+ synchronized Language getLanguage() throws LanguageNotFoundException {+ // synchronized to prevent multiple clients from trying to load the language at onceLanguageID id = description.getLanguageID();if (provider.isLanguageLoaded(id)) {// already loaded; no need to create a task- return provider.getLanguage(id);+ return provider.getLanguage(id, TaskMonitor.DUMMY);}+ AtomicReference<Throwable> langError = new AtomicReference<>();+ AtomicReference<Language> langResult = new AtomicReference<>();+//@formatter:off- TaskBuilder.withRunnable(monitor -> {- provider.getLanguage(id); // load and cache+ // Need to start task with canCancel true to get the cancel button added to the task dialog+ TaskBuilder.withRunnable(monitor -> {+ monitor.setCancelEnabled(false);+ try {+ langResult.set(provider.getLanguage(id, monitor));+ }+ catch (Throwable th) {+ langError.set(th);+ }})- .setTitle("Loading language '" + id + "'")- .setCanCancel(false)+ .setTitle("Loading language '%s'".formatted(id))+ .setCanCancel(true).setHasProgress(false)- .launchModal()- ;- //@formatter:on+ .launchModal();+ //@formatter:on++ Throwable th = langError.get();+ if (th instanceof LanguageNotFoundException lnfe) {+ throw lnfe;+ }+ else if (th != null) {+ throw new LanguageNotFoundException(id, th);+ }- return provider.getLanguage(id);+ return langResult.get();}@Overridediff --git a/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/JavaProcessBuilder.java b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/JavaProcessBuilder.javanew file mode 100644index 0000000000..fd6f65d3f0--- /dev/null+++ b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/JavaProcessBuilder.java@@ -0,0 +1,254 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.app.plugin.processors.sleigh;++import java.io.File;+import java.io.IOException;+import java.util.*;+import java.util.function.Consumer;++import generic.concurrent.io.ProcessConsumer;++/**+ * Helps creating and launching a java process.+ * <p>+ * By default, the launched process will have the same classpath as the current jvm and use the+ * same java binary to start the session.+ * <p>+ * Example usage:+ * <pre>+ * Process newproc = new JavaProcessBuilder(AClassWithMainEntryPoint.class)+ * .addProperty("sun.java2d.opengl", "false")+ * .addLaunchArg("-Xmx2G")+ * .withStdoutMonitor( (s) -> System.out.println(s) )+ * .start();+ * </pre>+ */+class JavaProcessBuilder {+ private static final String CP_SEP = System.getProperty("path.separator");++ private File javaBinary;+ private String mainClassname;+ private List<String> arguments;+ private String classPaths;+ private Map<String, String> javaProperties = new HashMap<>();+ private List<String> launchArgs = new ArrayList<>();+ private Consumer<String> stdoutMonitor;++ /**+ * Creates a java process builder, setting the main entry point for the launched process.+ *+ * @param mainClass class that contains a {@code public static void main(String[])} entry point.+ * Also implies that the launched process should have the same classpath as the current+ * jvm, or that the referenced class is somehow included in the launched process's classpath+ */+ public JavaProcessBuilder(Class<?> mainClass) {+ this.mainClassname = mainClass.getName();+ }++ /**+ * Creates a java process builder, setting the main entry point for the launched process.+ *+ * @param mainClassname name of a class that contains a+ * {@code public static void main(String[])} entry point.+ */+ public JavaProcessBuilder(String mainClassname) {+ this.mainClassname = mainClassname;+ }++ /**+ * Sets the arguments for the launched processes {@code main(String[] args)} method.+ *+ * @param newArguments arguments for the launched main() entry point+ * @return chainable ref to same+ */+ public JavaProcessBuilder withArguments(List<String> newArguments) {+ this.arguments = new ArrayList<>(newArguments);+ return this;+ }++ /**+ * Sets the location of the java jdk install, which controls how the bin/java[.exe] is found.+ *+ * @param javaHomeDir java home directory+ * @return chainable ref to same+ */+ public JavaProcessBuilder withJavaHome(File javaHomeDir) {+ this.javaBinary = javaHomeDir != null ? javaBinaryFromJavaHome(javaHomeDir) : null;++ return this;+ }++ private static File javaBinaryFromJavaHome(File javaHomeDir) {+ File binDir = new File(javaHomeDir, "bin");+ return new File(binDir, "java");+ }++ /**+ * Returns the location of the java binary that will be used to launch the new process.+ *+ * @return File pointing to the java jvm binary (java or java.exe)+ */+ public File getJavaBinary() {+ File f = javaBinary;+ if (f == null) {+ f = javaBinaryFromJavaHome(new File(System.getProperty("java.home")));+ }+ return f;+ }++ /**+ * Sets the classpaths for the launched process. The string is expected to be delimited with+ * the correct separators.+ *+ * @param newClassPaths classpath string for the launched process+ * @return chainable ref to same+ */+ public JavaProcessBuilder withClasspaths(String newClassPaths) {+ this.classPaths = newClassPaths;++ return this;+ }++ /**+ * Returns the classpath string that will be used to launch the new process.+ *+ * @return string+ */+ public String getClasspaths() {+ String s = classPaths;+ if (s == null) {+ s = System.getProperty("java.class.path");+ }+ return s;+ }++ /**+ * Adds an element to the classpath.+ *+ * @param classPath single classpath element+ * @return chainable ref to same+ */+ public JavaProcessBuilder addClasspath(String classPath) {+ this.classPaths = Objects.requireNonNullElse(this.classPaths, "") + CP_SEP + classPath;++ return this;+ }++ /**+ * Sets a callback that will handle each line written to stdout by the launched process.+ * <p>+ * The monitor will be called an additional time with a {@code null} value after the+ * spawned process has exited and its stdout has emptied.+ *+ * @param newStdoutMonitor Consumer<String> that will receive each text line written by+ * the launched process to it's stdout+ * @return chainable ref to same+ */+ public JavaProcessBuilder withStdoutMonitor(Consumer<String> newStdoutMonitor) {+ this.stdoutMonitor = newStdoutMonitor;++ return this;+ }++ /**+ * Adds a java system property to the launched process (e.g. "-DpropertyName=value").+ *+ * @param propertyName name of the property+ * @param propertyValue value of the property+ * @return chainable ref to same+ */+ public JavaProcessBuilder addProperty(String propertyName, String propertyValue) {+ javaProperties.put(propertyName, propertyValue);++ return this;+ }++ /**+ * Returns a list of properties that will be added to the launched process.+ *+ * @return list of property definition arguments+ */+ public List<String> getProperties() {+ return javaProperties.entrySet()+ .stream()+ .map(entry -> "-D%s=%s".formatted(entry.getKey(), entry.getValue()))+ .toList();+ }++ /**+ * Adds a java launch argument (e.g. "-Xmx512M", etc)+ *+ * @param launchArg raw argument to pass to the java binary when launching+ * @return chainable ref to same+ */+ public JavaProcessBuilder addLaunchArg(String launchArg) {+ launchArgs.add(launchArg);++ return this;+ }++ /**+ * Creates a 'real' {@link ProcessBuilder} using the information specified in this builder.+ * <p>+ * The stdout monitor will still need to be installed into any launched process.+ *+ * @return {@link ProcessBuilder}+ */+ public ProcessBuilder getProcessBuilder() {+ Objects.requireNonNull(mainClassname);++ List<String> commandParts = new ArrayList<>();+ commandParts.add(getJavaBinary().getPath());+ commandParts.addAll(launchArgs);+ String cp = getClasspaths();+ if (!cp.isBlank()) {+ commandParts.add("-cp");+ commandParts.add(cp);+ }+ commandParts.addAll(getProperties());+ commandParts.add(mainClassname);+ commandParts.addAll(arguments != null ? arguments : List.of());++ ProcessBuilder pb = new ProcessBuilder(commandParts);+ return pb;+ }++ /**+ * Installs a monitor that reads the processes stdout+ *+ * @param process {@link Process}+ */+ public void installMonitor(Process process) {+ ProcessConsumer.monitorAndSignalEof(process.getInputStream(), stdoutMonitor,+ "stdout[%d]".formatted(process.pid()));+ }++ /**+ * Launches a new java process using the information specified in this builder.+ *+ * @return {@link Process}+ * @throws IOException if error starting process+ */+ public Process start() throws IOException {+ ProcessBuilder pb = getProcessBuilder();+ Process process = pb.start();+ if (stdoutMonitor != null) {+ installMonitor(process);+ }+ return process;+ }+}diff --git a/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProviderTest.java b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProviderTest.javanew file mode 100644index 0000000000..c6a8728d95--- /dev/null+++ b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProviderTest.java@@ -0,0 +1,230 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.app.plugin.processors.sleigh;++import static org.junit.Assert.*;++import java.io.File;+import java.io.IOException;+import java.time.Duration;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicInteger;++import org.junit.Test;++import generic.jar.ResourceFile;+import generic.test.AbstractGenericTest;+import ghidra.framework.Application;+import ghidra.program.model.lang.Language;+import ghidra.program.model.lang.LanguageID;+import ghidra.util.Msg;+import ghidra.util.SystemUtilities;+import ghidra.util.task.TaskMonitor;++public class SleighLanguageProviderTest extends AbstractGenericTest {++ private LanguageID x86LangId = new LanguageID("x86:LE:32:default");+ private ResourceFile x86LdefsFile = Application.findDataFileInAnyModule("languages/x86.ldefs");+ private SleighLanguageProvider langProvider;+++ @Test(timeout = 60000 + 5000 /* 1 minute (default lock timeout) + 5 seconds */)+ public void testSlaThunderingHerds() throws Exception {+ // Tests when a thundering herd of processes all try to read a sleigh file at the same time++ langProvider = new SleighLanguageProvider(x86LdefsFile);++ // Ensure the lang file exist, and then tweak the timestamp to force one of the+ // spawned procs to recompile the sla+ SleighLanguage lang = langProvider.getLanguage(x86LangId, TaskMonitor.DUMMY);+ SleighLanguageDescription langDesc = lang.getLanguageDescription();+ SleighLanguageFile langFile = langDesc.getLanguageFile();+ File slaFile = langFile.getSlaFile().getFile(false);+ File slaSpecFile = langFile.getSlaSpecFile().getFile(false);+ slaFile.setLastModified(slaSpecFile.lastModified() - 60 * 1000);++ int procsToStart = 20;+ long langProviderTimeout = SleighLanguageProvider.LANGUAGE_LOCK_TIMEOUT.toMillis();+ long test_start = System.currentTimeMillis();++ // 1) start a lot of processes+ // 2) wait until they initialize themselves and get to step1 (they wait also for next step)+ // 3) allow all launched processes to proceed at same time+ // 4) wait for all to exit with success exit code+ List<TestProc> procs = new ArrayList<>();+ for (int procNum = 0; procNum < procsToStart; procNum++) {+ procs.add(TestProc.start(procNum, langProviderTimeout));+ }++ for (TestProc testProc : procs) {+ if (!testProc.waitUntilStep(1)) {+ fail(testProc.msg("failed to reach step1"));+ }+ }++ // If the spawned processes are writing to the .sla file incorrectly and simultaneously,+ // giving a small delay between each process's critical action historically help expose+ // the error.+ int DELAY_BETWEEN_PROCESSES = 600;+ for (TestProc testProc : procs) {+ testProc.sendGoahead();+ sleep(DELAY_BETWEEN_PROCESSES);+ }++ int badExitCount = 0;+ for (TestProc testProc : procs) {+ if (!testProc.waitUntilStep(2)) {+ testProc.log("exited before step 2 reached");+ badExitCount++;+ }+ int exitCode = testProc.proc.waitFor();+ if (exitCode != 0) {+ testProc.log("exited with error: %d".formatted(exitCode));+ badExitCount++;+ }+ }++ Msg.info(this, "Total test time: " + (System.currentTimeMillis() - test_start));++ assertTrue(badExitCount == 0);+ }++ @Test(timeout = 10000 + 5000 /* shorterTimeout + 5 seconds */)+ public void testSlaLockTimeout() throws Exception {+ // Test that lock timeout successfully causes a failure+ langProvider = new SleighLanguageProvider(x86LdefsFile);++ SleighLanguageDescription langDesc = langProvider.getLanguageDescription(x86LangId);+ SleighLanguageFile langFile = langDesc.getLanguageFile();++ long shorterTimeoutMS = Duration.ofSeconds(10).toMillis();++ TestProc testProc = TestProc.start(0, shorterTimeoutMS);+ assertTrue(testProc.waitUntilStep(1));+ langFile.withLock(Duration.ofMillis(10), TaskMonitor.DUMMY, () -> {+ testProc.sendGoahead();+ Thread.sleep(shorterTimeoutMS + 1000); // hold lock while testProc is trying to get it, force it to fail+ });++ testProc.assertExitNum(1);+ }++ public void testSlaLanguage_FromOtherProcess() throws Throwable {+ // this is not a junit test entry point, but instead is what is run+ // in each sub-process launched by each TestProc instance.++ langProvider = new SleighLanguageProvider(x86LdefsFile);+ TestProc.waitForGoahead();+ Language lang = langProvider.getLanguage(x86LangId, TaskMonitor.DUMMY);++ // write via stdout to the parent process, which will handle logging+ System.out.println("STEP 2 Got language %s".formatted(lang));+ System.out.flush();++ System.exit(0);+ }++ public static void main(String[] args) {+ // this is the entry point for the launched TestProcs that will be fighting over+ // a sleigh language file+ try {+ SleighLanguageProviderTest test = new SleighLanguageProviderTest();+ test.testSlaLanguage_FromOtherProcess();+ }+ catch (Throwable th) {+ // write via stdout to the parent process, which will handle logging+ System.out.println("Exception " + th);+ th.printStackTrace(System.out);+ System.exit(1);+ }+ }++ /**+ * Handles coordinating an external java process that will be attempting to access a common+ * sleigh language file.+ */+ static class TestProc {+ static TestProc start(int procNum, long timeoutMS) throws IOException {+ TestProc testProc = new TestProc();+ testProc.procNum = procNum;+ testProc.proc = new JavaProcessBuilder(SleighLanguageProviderTest.class) // self class's main()+ .addProperty(SystemUtilities.TESTING_PROPERTY, "true")+ .addProperty(SleighLanguageProvider.LANGUAGE_LOCK_TIMEOUT_PROPNAME,+ "" + timeoutMS)+ .withStdoutMonitor(testProc::update)+ .start();+ testProc.log("Started");+ return testProc;+ }++ Process proc;+ int procNum;+ AtomicInteger stepNum = new AtomicInteger();+ AtomicBoolean stdoutMonitorDone = new AtomicBoolean();+ Thread monitorThread;++ void sendGoahead() throws IOException {+ proc.getOutputStream().write('\n');+ proc.getOutputStream().flush();+ }++ static void waitForGoahead() throws IOException {+ // write via stdout to the parent process, which will handle logging+ System.out.println("STEP 1 Waiting for synchronization go-ahead...");+ System.out.flush();+ int b = System.in.read();+ System.out.println("Read byte: " + b);+ System.out.flush();+ }++ boolean waitUntilStep(int waitForStepNum) throws InterruptedException {+ while (stepNum.get() < waitForStepNum) {+ if (stdoutMonitorDone.get()) {+ return false;+ }+ Thread.sleep(200);+ }+ return true;+ }++ void update(String s) {+ if (s == null) {+ stdoutMonitorDone.set(true);+ return;+ }+ log(s);+ if (s.startsWith("STEP ")) {+ stepNum.set(Integer.parseInt(s.split(" ")[1]));+ }+ }++ void assertExitNum(int exitNum) throws InterruptedException {+ assertEquals(exitNum, proc.waitFor()); // wait until helper process has finished+ log("exit code: " + proc.waitFor());+ }++ String msg(String s) {+ return "TestProc[%d-%d] %s".formatted(procNum, proc.pid(), s);+ }++ void log(String s) {+ Msg.info(this, msg(s));+ }+ }++}diff --git a/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/X86SleighLanguageLockerTest.java b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/X86SleighLanguageLockerTest.javanew file mode 100644index 0000000000..3c75e270d0--- /dev/null+++ b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/X86SleighLanguageLockerTest.java@@ -0,0 +1,58 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.app.plugin.processors.sleigh;++import java.time.Duration;++import org.junit.Test;++import generic.jar.ResourceFile;+import ghidra.GhidraApplicationLayout;+import ghidra.framework.Application;+import ghidra.framework.ApplicationConfiguration;+import ghidra.program.model.lang.LanguageID;+import ghidra.util.task.TaskMonitor;++public class X86SleighLanguageLockerTest {++ private LanguageID x86LangId = new LanguageID("x86:LE:32:default");++ //@Test+ public void testLockLanguageForever() throws Exception {+ // Locks the language's file forever, for manually testing the GUI frontend when it tries+ // to load a language.+ // This test should not be enabled by default.++ // Don't use a test application config because we need to use the same user specific+ // .ghidra/.ghidra-ver config directories+ Application.initializeApplication(new GhidraApplicationLayout(),+ new ApplicationConfiguration());++ ResourceFile x86LdefsFile = Application.findDataFileInAnyModule("languages/x86.ldefs");+ SleighLanguageProvider langProvider = new SleighLanguageProvider(x86LdefsFile);++ SleighLanguageDescription langDesc = langProvider.getLanguageDescription(x86LangId);+ SleighLanguageFile langFile = langDesc.getLanguageFile();+ // we are reading from stdin, so should use stdout to prompt the user+ System.out.println("Locking lang file: " + langFile.getSlaFile());++ langFile.withLock(Duration.ofMillis(10), TaskMonitor.DUMMY, () -> {+ System.out.println("Press enter to end lock");+ System.in.read(); // if user hits enter in the console of the test, will exit+ });+ }++}diff --git a/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptionsTest.java b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptionsTest.javanew file mode 100644index 0000000000..b6936747c7--- /dev/null+++ b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptionsTest.java@@ -0,0 +1,150 @@+/* ###+ * IP: GHIDRA+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package ghidra.pcodeCPort.slgh_compile;++import static org.junit.Assert.*;++import java.io.File;+import java.io.IOException;++import org.junit.Test;++import generic.test.AbstractGenericTest;+import utilities.util.FileUtilities;++public class SleighCompileOptionsTest extends AbstractGenericTest {+ @Test+ public void testArgFile() throws IOException {+ File argTmp = createTempFile("sleigh_args");+ FileUtilities.writeStringToFile(argTmp, """+ -x+ -u+ -l+ -n+ -t+ -e+ -c+ -f+ -s+ -DTESTNAME=TESTVALUE+ -dx86+ """);++ // test default -x before doing any option parsing because its a global flag+ assertFalse(SleighCompile.yydebug);++ SleighCompileOptions defaultOpts = new SleighCompileOptions();+ SleighCompileOptions opts = SleighCompileOptions.fromFile(argTmp);++ // -x+ assertTrue(SleighCompile.yydebug);++ // -u+ assertTrue(opts.unnecessaryPcodeWarning);+ assertNotEquals(defaultOpts.unnecessaryPcodeWarning, opts.unnecessaryPcodeWarning);++ // -l+ assertFalse(opts.lenientConflict);+ assertNotEquals(defaultOpts.lenientConflict, opts.lenientConflict);++ // -n+ assertTrue(opts.allNopWarning);+ assertNotEquals(defaultOpts.allNopWarning, opts.allNopWarning);++ // -t+ assertTrue(opts.deadTempWarning);+ assertNotEquals(defaultOpts.deadTempWarning, opts.deadTempWarning);++ // -e+ assertTrue(opts.enforceLocalKeyWord);+ assertNotEquals(defaultOpts.enforceLocalKeyWord, opts.enforceLocalKeyWord);++ // -c+ assertTrue(opts.allCollisionWarning);+ assertNotEquals(defaultOpts.allCollisionWarning, opts.allCollisionWarning);++ // -f+ assertTrue(opts.unusedFieldWarning);+ assertNotEquals(defaultOpts.unusedFieldWarning, opts.unusedFieldWarning);++ // -s+ assertTrue(opts.caseSensitiveRegisterNames);+ assertNotEquals(defaultOpts.caseSensitiveRegisterNames, opts.caseSensitiveRegisterNames);++ // -D+ assertNull(defaultOpts.preprocs.get("TESTNAME"));+ assertEquals("TESTVALUE", opts.preprocs.get("TESTNAME"));++ // -d+ assertNull(defaultOpts.preprocs.get("x86"));+ assertNotNull(opts.preprocs.get("x86"));+ assertTrue(new File(opts.preprocs.get("x86")).isDirectory());+ }++ @Test+ public void testCmdLineArgs_1input() throws IOException {+ File argTmp = createTempFile("sleigh_args");+ FileUtilities.writeStringToFile(argTmp, "-DTESTNAME=TESTVALUE");++ String[] args = { "-i", argTmp.getPath(), "inputfile.slaspec" };+ SleighCompileOptions opts = SleighCompileOptions.parse(args);++ assertEquals("TESTVALUE", opts.preprocs.get("TESTNAME"));+ assertEquals(opts.inputFile.getName(), "inputfile.slaspec");+ assertEquals(opts.outputFile.getName(), "inputfile.sla");+ }++ @Test+ public void testCmdLineArgs_1input_default_ext() throws IOException {+ File argTmp = createTempFile("sleigh_args");+ FileUtilities.writeStringToFile(argTmp, "-DTESTNAME=TESTVALUE");++ String[] args = { "-i", argTmp.getPath(), "inputfile" };+ SleighCompileOptions opts = SleighCompileOptions.parse(args);++ assertEquals("TESTVALUE", opts.preprocs.get("TESTNAME"));+ assertEquals(opts.inputFile.getName(), "inputfile.slaspec");+ assertEquals(opts.outputFile.getName(), "inputfile.sla");+ }++ @Test+ public void testCmdLineArgs_input_and_output() throws IOException {+ File argTmp = createTempFile("sleigh_args");+ FileUtilities.writeStringToFile(argTmp, "-DTESTNAME=TESTVALUE");++ String[] args = { "-i", argTmp.getPath(), "inputfile.slaspec", "outputfile.sla" };+ SleighCompileOptions opts = SleighCompileOptions.parse(args);++ assertEquals("TESTVALUE", opts.preprocs.get("TESTNAME"));+ assertEquals(opts.inputFile.getName(), "inputfile.slaspec");+ assertEquals(opts.outputFile.getName(), "outputfile.sla");+ }++ @Test+ public void testCmdLineArgs_all() throws IOException {+ File argTmp = createTempFile("sleigh_args");+ FileUtilities.writeStringToFile(argTmp, "-DTESTNAME=TESTVALUE");+ File allDir = createTempDirectory("alldir");++ String[] args = { "-i", argTmp.getPath(), "-a", allDir.getPath() };+ SleighCompileOptions opts = SleighCompileOptions.parse(args);++ assertEquals("TESTVALUE", opts.preprocs.get("TESTNAME"));+ assertTrue(opts.allMode);+ assertEquals(opts.allDir.getPath(), allDir.getPath());+ }++}