Ghidra is a software reverse engineering (SRE) framework
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package agent.dbgeng.rmi;
import static org.junit.Assert.*;
import static org.junit.Assume.*;
import java.io.*;
import java.net.*;
import java.nio.file.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.function.Function;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.junit.Before;
import org.junit.BeforeClass;
import ghidra.app.plugin.core.debug.gui.AbstractGhidraHeadedDebuggerTest;
import ghidra.app.plugin.core.debug.service.tracermi.TraceRmiPlugin;
import ghidra.app.plugin.core.debug.utils.ManagedDomainObject;
import ghidra.app.services.TraceRmiService;
import ghidra.debug.api.tracermi.*;
import ghidra.framework.*;
import ghidra.framework.main.ApplicationLevelOnlyPlugin;
import ghidra.framework.model.DomainFile;
import ghidra.framework.plugintool.Plugin;
import ghidra.framework.plugintool.PluginsConfiguration;
import ghidra.framework.plugintool.util.*;
import ghidra.program.model.address.Address;
import ghidra.program.model.address.AddressRangeImpl;
import ghidra.pty.testutil.DummyProc;
import ghidra.trace.model.Lifespan;
import ghidra.trace.model.Trace;
import ghidra.trace.model.breakpoint.TraceBreakpointKind;
import ghidra.trace.model.breakpoint.TraceBreakpointKind.TraceBreakpointKindSet;
import ghidra.trace.model.target.TraceObject;
import ghidra.trace.model.target.TraceObjectValue;
import ghidra.util.*;
public abstract class AbstractDbgEngTraceRmiTest extends AbstractGhidraHeadedDebuggerTest {
/**
* Some features have to be disabled to avoid permissions issues in the test container. Namely,
* don't try to disable ASLR.
*/
public static final String PREAMBLE = """
from ghidradbg.commands import *
from ghidratrace.client import Schedule
""";
// Connecting should be the first thing the script does, so use a tight timeout.
protected static final int CONNECT_TIMEOUT_MS = 6000;
protected static final int TIMEOUT_SECONDS = SystemUtilities.isInTestingBatchMode() ? 20 : 300;
protected static final int QUIT_TIMEOUT_MS = 2000;
/** Some snapshot likely to exceed the latest */
protected static final long SNAP = 100;
protected static boolean didSetupPython = false;
public static final String INSTRUMENT_STATE = """
import sys
from comtypes.hresult import S_OK
from ghidradbg import commands, hooks
print("Instrumenting")
@hooks.log_errors
def on_state_changed(*args):
if args[0] != DbgEng.DEBUG_CES_EXECUTION_STATUS:
return S_OK
print("State changed: {:x}".format(args[1]))
sys.stdout.flush()
proc = util.selected_process()
trace = commands.STATE.trace
with commands.STATE.client.batch():
with trace.open_tx("State changed proc {}".format(proc)):
commands.put_state(proc)
return S_OK
# Without this, the engine seems to GO after the interrupt
@hooks.log_errors
def on_exception(*args):
return DbgEng.DEBUG_STATUS_BREAK
@util.dbg.eng_thread
def install_hooks():
print("Installing")
util.dbg._base.events.engine_state(handler=on_state_changed)
util.dbg._base.events.exception(handler=on_exception)
install_hooks()
""";
protected TraceRmiService traceRmi;
private Path pythonPath;
private Path outFile;
private Path errFile;
@BeforeClass
public static void setupPython() throws Throwable {
if (didSetupPython) {
// Only do this once when running the full suite.
return;
}
if (SystemUtilities.isInTestingBatchMode()) {
// Don't run gradle in gradle. It already did this task.
return;
}
String gradle = DummyProc.which("gradle.bat");
new ProcessBuilder(gradle, "assemblePyPackage")
.directory(TestApplicationUtils.getInstallationDirectory())
.inheritIO()
.start()
.waitFor();
didSetupPython = true;
}
protected void setPythonPath(ProcessBuilder pb) throws IOException {
String sep =
OperatingSystem.CURRENT_OPERATING_SYSTEM == OperatingSystem.WINDOWS ? ";" : ":";
String rmiPyPkg = Application.getModuleSubDirectory("Debugger-rmi-trace",
"build/pypkg/src").getAbsolutePath();
String gdbPyPkg = Application.getModuleSubDirectory("Debugger-agent-dbgeng",
"build/pypkg/src").getAbsolutePath();
String add = rmiPyPkg + sep + gdbPyPkg;
pb.environment().compute("PYTHONPATH", (k, v) -> v == null ? add : (v + sep + add));
}
protected void setWindbgPath(ProcessBuilder pb) throws IOException {
pb.environment().put("WINDBG_DIR", "C:\\Program Files\\Amazon Corretto\\jdk21.0.3_9\\bin");
}
@BeforeClass
public static void assertOS() {
assumeTrue("Not on Windows",
OperatingSystem.CURRENT_OPERATING_SYSTEM == OperatingSystem.WINDOWS);
}
@Before
public void setupTraceRmi() throws Throwable {
traceRmi = addPlugin(tool, TraceRmiPlugin.class);
pythonPath = getPathToPython();
assertTrue(pythonPath.toFile().exists());
outFile = Files.createTempFile("pydbgout", null);
errFile = Files.createTempFile("pydbgerr", null);
}
protected Path getPathToPython() {
try {
String py3path = DummyProc.which("python3");
if (py3path != null && !py3path.contains("msys")) {
return Paths.get(py3path);
}
}
catch (RuntimeException e) {
// Should not happen since we maintain library on host test machine
}
return Paths.get(DummyProc.which("python"));
}
protected void addAllDebuggerPlugins() throws PluginException {
PluginsConfiguration plugConf = new PluginsConfiguration() {
@Override
protected boolean accepts(Class<? extends Plugin> pluginClass) {
return !ApplicationLevelOnlyPlugin.class.isAssignableFrom(pluginClass);
}
};
for (PluginDescription pd : plugConf
.getPluginDescriptions(PluginPackage.getPluginPackage("Debugger"))) {
addPlugin(tool, pd.getPluginClass());
}
}
protected static String addrToStringForPython(InetAddress address) {
if (address.isAnyLocalAddress()) {
return "127.0.0.1"; // Can't connect to 0.0.0.0 as such. Choose localhost.
}
return address.getHostAddress();
}
protected static String sockToStringForPython(SocketAddress address) {
if (address instanceof InetSocketAddress tcp) {
return addrToStringForPython(tcp.getAddress()) + ":" + tcp.getPort();
}
throw new AssertionError("Unhandled address type " + address);
}
protected record PythonResult(boolean timedOut, int exitCode, String stdout, String stderr) {
protected String handle() {
if (stderr.contains("Error") || (0 != exitCode && 1 != exitCode && 143 != exitCode)) {
throw new PythonError(exitCode, stdout, stderr);
}
return stdout;
}
}
protected record ExecInPython(Process python, CompletableFuture<PythonResult> future) {}
protected void pump(InputStream streamIn, OutputStream streamOut) {
Thread t = new Thread(() -> {
try (PrintStream printOut = new PrintStream(streamOut);
BufferedReader reader = new BufferedReader(new InputStreamReader(streamIn))) {
String line;
while ((line = reader.readLine()) != null) {
printOut.println(line);
printOut.flush();
}
}
catch (IOException e) {
Msg.info(this, "Terminating stdin pump, because " + e);
}
});
t.setDaemon(true);
t.start();
}
protected void pumpTee(InputStream streamIn, File fileOut, PrintStream streamOut) {
Thread t = new Thread(() -> {
try (PrintStream fileStream = new PrintStream(fileOut);
BufferedReader reader = new BufferedReader(new InputStreamReader(streamIn))) {
String line;
while ((line = reader.readLine()) != null) {
streamOut.println(line);
streamOut.flush();
fileStream.println(line);
fileStream.flush();
}
}
catch (IOException e) {
Msg.info(this, "Terminating tee: " + fileOut + ", because " + e);
}
});
t.setDaemon(true);
t.start();
}
@SuppressWarnings("resource") // Do not close stdin
protected ExecInPython execInPython(String script) throws IOException {
ProcessBuilder pb = new ProcessBuilder(pythonPath.toString(), "-i");
setPythonPath(pb);
setWindbgPath(pb);
// If commands come from file, Python will quit after EOF.
Msg.info(this, "outFile: " + outFile);
Msg.info(this, "errFile: " + errFile);
//pb.inheritIO();
pb.redirectInput(ProcessBuilder.Redirect.PIPE);
if (SystemUtilities.isInTestingBatchMode()) {
pb.redirectOutput(outFile.toFile());
pb.redirectError(errFile.toFile());
}
else {
pb.redirectOutput(ProcessBuilder.Redirect.PIPE);
pb.redirectError(ProcessBuilder.Redirect.PIPE);
}
Process pyproc = pb.start();
if (!SystemUtilities.isInTestingBatchMode()) {
pumpTee(pyproc.getInputStream(), outFile.toFile(), System.out);
pumpTee(pyproc.getErrorStream(), errFile.toFile(), System.err);
}
OutputStream stdin = pyproc.getOutputStream();
stdin.write(script.getBytes());
stdin.flush();
if (!SystemUtilities.isInTestingBatchMode()) {
pump(System.in, stdin);
}
return new ExecInPython(pyproc, CompletableFuture.supplyAsync(() -> {
try {
if (!pyproc.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
Msg.error(this, "Timed out waiting for Python");
pyproc.destroyForcibly();
pyproc.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS);
return new PythonResult(true, -1, Files.readString(outFile),
Files.readString(errFile));
}
Msg.info(this, "Python exited with code " + pyproc.exitValue());
return new PythonResult(false, pyproc.exitValue(), Files.readString(outFile),
Files.readString(errFile));
}
catch (Exception e) {
return ExceptionUtils.rethrow(e);
}
finally {
pyproc.destroyForcibly();
}
}));
}
public static class PythonError extends RuntimeException {
public final int exitCode;
public final String stdout;
public final String stderr;
public PythonError(int exitCode, String stdout, String stderr) {
super("""
exitCode=%d:
----stdout----
%s
----stderr----
%s
""".formatted(exitCode, stdout, stderr));
this.exitCode = exitCode;
this.stdout = stdout;
this.stderr = stderr;
}
}
protected String runThrowError(String script) throws Exception {
CompletableFuture<PythonResult> result = execInPython(script).future;
return result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
}
protected record PythonAndConnection(ExecInPython exec, TraceRmiConnection connection)
implements AutoCloseable {
protected RemoteMethod getMethod(String name) {
return Objects.requireNonNull(connection.getMethods().get(name));
}
public void execute(String cmd) {
RemoteMethod execute = getMethod("execute");
execute.invoke(Map.of("cmd", cmd));
}
public RemoteAsyncResult executeAsync(String cmd) {
RemoteMethod execute = getMethod("execute");
return execute.invokeAsync(Map.of("cmd", cmd));
}
public String executeCapture(String cmd) {
RemoteMethod execute = getMethod("execute");
return (String) execute.invoke(Map.of("cmd", cmd, "to_string", true));
}
@Override
public void close() throws Exception {
Msg.info(this, "Cleaning up python");
exec.python().destroy();
try {
PythonResult r = exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
r.handle();
waitForPass(this, () -> assertTrue(connection.isClosed()),
TIMEOUT_SECONDS, TimeUnit.SECONDS);
}
finally {
exec.python.destroyForcibly();
}
}
}
protected PythonAndConnection startAndConnectPython(Function<String, String> scriptSupplier)
throws Exception {
TraceRmiAcceptor acceptor = traceRmi.acceptOne(null);
ExecInPython exec =
execInPython(scriptSupplier.apply(sockToStringForPython(acceptor.getAddress())));
acceptor.setTimeout(CONNECT_TIMEOUT_MS);
try {
TraceRmiConnection connection = acceptor.accept();
return new PythonAndConnection(exec, connection);
}
catch (SocketTimeoutException e) {
exec.python.destroyForcibly();
exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
throw e;
}
}
protected PythonAndConnection startAndConnectPython() throws Exception {
return startAndConnectPython(addr -> """
%s
ghidra_trace_connect('%s')
""".formatted(PREAMBLE, addr));
}
@SuppressWarnings("resource")
protected String runThrowError(Function<String, String> scriptSupplier)
throws Exception {
PythonAndConnection conn = startAndConnectPython(scriptSupplier);
PythonResult r = conn.exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
String stdout = r.handle();
waitForPass(this, () -> assertTrue(conn.connection.isClosed()),
TIMEOUT_SECONDS, TimeUnit.SECONDS);
return stdout;
}
protected void waitStopped(String message) {
TraceObject proc =
Objects.requireNonNull(tb.objAny("Sessions[].Processes[]", Lifespan.at(0)));
waitForPass(() -> assertEquals(message, "STOPPED", tb.objValue(proc, 0, "_state")));
waitTxDone();
}
protected void waitRunning(String message) {
TraceObject proc =
Objects.requireNonNull(tb.objAny("Sessions[].Processes[]", Lifespan.at(0)));
waitForPass(() -> assertEquals(message, "RUNNING", tb.objValue(proc, 0, "_state")));
waitTxDone();
}
protected String extractOutSection(String out, String head) {
String[] split = out.split("\n");
String xout = "";
for (String s : split) {
if (!s.startsWith("(python)") && !s.equals("")) {
xout += s + "\n";
}
}
return xout.split(head)[1].split("---")[0].replace("(python)", "").trim();
}
record MemDump(long address, byte[] data) {}
protected MemDump parseHexDump(String dump) throws IOException {
// First, get the address. Assume contiguous, so only need top line.
List<String> lines = List.of(dump.split("\n"));
List<String> toksLine0 = List.of(lines.get(0).split("\\s+"));
String addrstr = toksLine0.get(0);
if (addrstr.contains(":")) {
addrstr = addrstr.substring(0, addrstr.indexOf(":"));
}
long address = Long.parseLong(addrstr, 16);
ByteArrayOutputStream buf = new ByteArrayOutputStream();
for (String l : lines) {
List<String> parts = List.of(l.split(":"));
assertEquals(2, parts.size());
String hex = parts.get(1).substring(0, 48);
byte[] lineData = NumericUtilities.convertStringToBytes(hex);
assertNotNull("Converted to null: " + hex, parts.get(1));
buf.write(lineData);
}
return new MemDump(address, buf.toByteArray());
}
record RegDump() {}
protected RegDump parseRegDump(String dump) {
return new RegDump();
}
protected ManagedDomainObject<Trace> openTrace(String path) throws Exception {
DomainFile df = env.getProject().getProjectData().getFile(path);
assertNotNull(df);
return new ManagedDomainObject<>(df, Trace.class, monitor);
}
protected ManagedDomainObject<Trace> waitTrace(String path) throws Exception {
DomainFile df;
long start = System.currentTimeMillis();
while (true) {
df = env.getProject().getProjectData().getFile(path);
if (df != null) {
return new ManagedDomainObject<>(df, Trace.class, monitor);
}
Thread.sleep(1000);
if (System.currentTimeMillis() - start > 30000) {
throw new TimeoutException("30 seconds expired waiting for domain file");
}
}
}
protected void assertBreakLoc(TraceObjectValue locVal, String key, Address addr, int len,
Set<TraceBreakpointKind> kinds, String expression) throws Exception {
assertEquals(key, locVal.getEntryKey());
TraceObject loc = locVal.getChild();
TraceObject spec = loc;
assertEquals(new AddressRangeImpl(addr, len), loc.getValue(0, "_range").getValue());
assertEquals(TraceBreakpointKindSet.encode(kinds), spec.getValue(0, "_kinds").getValue());
assertTrue(spec.getValue(0, "_expression").getValue().toString().contains(expression));
}
protected void assertWatchLoc(TraceObjectValue locVal, String key, Address addr, int len,
Set<TraceBreakpointKind> kinds, String expression) throws Exception {
assertEquals(key, locVal.getEntryKey());
TraceObject loc = locVal.getChild();
assertEquals(new AddressRangeImpl(addr, len), loc.getValue(0, "_range").getValue());
assertEquals(TraceBreakpointKindSet.encode(kinds), loc.getValue(0, "_kinds").getValue());
}
protected void waitTxDone() {
waitFor(() -> tb.trace.getCurrentTransactionInfo() == null);
}
public static void waitForPass(Runnable runnable, long timeoutMs, long retryDelayMs) {
long start = System.currentTimeMillis();
AssertionError lastError = null;
while (System.currentTimeMillis() - start < timeoutMs) {
try {
runnable.run();
return;
}
catch (AssertionError e) {
lastError = e;
}
try {
Thread.sleep(retryDelayMs);
}
catch (InterruptedException e) {
// Retry sooner, I guess.
}
}
if (lastError == null) {
throw new AssertionError("Timed out before first try?");
}
throw lastError;
}
}