Skip to content

Ghidra is a software reverse engineering (SRE) framework

16.9 KB
Raw Blame
/* ###
 * IP: GHIDRA
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package agent.dbgeng.rmi;

import static org.junit.Assert.*;
import static org.junit.Assume.*;

import java.io.*;
import java.net.*;
import java.nio.file.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.function.Function;

import org.apache.commons.lang3.exception.ExceptionUtils;
import org.junit.Before;
import org.junit.BeforeClass;

import ghidra.app.plugin.core.debug.gui.AbstractGhidraHeadedDebuggerTest;
import ghidra.app.plugin.core.debug.service.tracermi.TraceRmiPlugin;
import ghidra.app.plugin.core.debug.utils.ManagedDomainObject;
import ghidra.app.services.TraceRmiService;
import ghidra.debug.api.tracermi.*;
import ghidra.framework.*;
import ghidra.framework.main.ApplicationLevelOnlyPlugin;
import ghidra.framework.model.DomainFile;
import ghidra.framework.plugintool.Plugin;
import ghidra.framework.plugintool.PluginsConfiguration;
import ghidra.framework.plugintool.util.*;
import ghidra.program.model.address.Address;
import ghidra.program.model.address.AddressRangeImpl;
import ghidra.pty.testutil.DummyProc;
import ghidra.trace.model.Lifespan;
import ghidra.trace.model.Trace;
import ghidra.trace.model.breakpoint.TraceBreakpointKind;
import ghidra.trace.model.breakpoint.TraceBreakpointKind.TraceBreakpointKindSet;
import ghidra.trace.model.target.TraceObject;
import ghidra.trace.model.target.TraceObjectValue;
import ghidra.util.*;

public abstract class AbstractDbgEngTraceRmiTest extends AbstractGhidraHeadedDebuggerTest {
	/**
	 * Some features have to be disabled to avoid permissions issues in the test container. Namely,
	 * don't try to disable ASLR.
	 */
	public static final String PREAMBLE = """
			from ghidradbg.commands import *
			from ghidratrace.client import Schedule
			""";
	// Connecting should be the first thing the script does, so use a tight timeout.
	protected static final int CONNECT_TIMEOUT_MS = 6000;
	protected static final int TIMEOUT_SECONDS = SystemUtilities.isInTestingBatchMode() ? 20 : 300;
	protected static final int QUIT_TIMEOUT_MS = 2000;

	/** Some snapshot likely to exceed the latest */
	protected static final long SNAP = 100;

	protected static boolean didSetupPython = false;

	public static final String INSTRUMENT_STATE = """
			import sys
			from comtypes.hresult import S_OK
			from ghidradbg import commands, hooks
			print("Instrumenting")
			@hooks.log_errors
			def on_state_changed(*args):
			    if args[0] != DbgEng.DEBUG_CES_EXECUTION_STATUS:
			        return S_OK
			    print("State changed: {:x}".format(args[1]))
			    sys.stdout.flush()
			    proc = util.selected_process()
			    trace = commands.STATE.trace
			    with commands.STATE.client.batch():
			        with trace.open_tx("State changed proc {}".format(proc)):
			            commands.put_state(proc)
			    return S_OK

			# Without this, the engine seems to GO after the interrupt
			@hooks.log_errors
			def on_exception(*args):
			    return DbgEng.DEBUG_STATUS_BREAK

			@util.dbg.eng_thread
			def install_hooks():
			    print("Installing")
			    util.dbg._base.events.engine_state(handler=on_state_changed)
			    util.dbg._base.events.exception(handler=on_exception)

			install_hooks()
			""";

	protected TraceRmiService traceRmi;
	private Path pythonPath;
	private Path outFile;
	private Path errFile;

	@BeforeClass
	public static void setupPython() throws Throwable {
		if (didSetupPython) {
			// Only do this once when running the full suite.
			return;
		}
		if (SystemUtilities.isInTestingBatchMode()) {
			// Don't run gradle in gradle. It already did this task.
			return;
		}
		String gradle = DummyProc.which("gradle.bat");
		new ProcessBuilder(gradle, "assemblePyPackage")
				.directory(TestApplicationUtils.getInstallationDirectory())
				.inheritIO()
				.start()
				.waitFor();
		didSetupPython = true;
	}

	protected void setPythonPath(ProcessBuilder pb) throws IOException {
		String sep =
			OperatingSystem.CURRENT_OPERATING_SYSTEM == OperatingSystem.WINDOWS ? ";" : ":";
		String rmiPyPkg = Application.getModuleSubDirectory("Debugger-rmi-trace",
			"build/pypkg/src").getAbsolutePath();
		String gdbPyPkg = Application.getModuleSubDirectory("Debugger-agent-dbgeng",
			"build/pypkg/src").getAbsolutePath();
		String add = rmiPyPkg + sep + gdbPyPkg;
		pb.environment().compute("PYTHONPATH", (k, v) -> v == null ? add : (v + sep + add));
	}

	protected void setWindbgPath(ProcessBuilder pb) throws IOException {
		pb.environment().put("WINDBG_DIR", "C:\\Program Files\\Amazon Corretto\\jdk21.0.3_9\\bin");
	}

	@BeforeClass
	public static void assertOS() {
		assumeTrue("Not on Windows",
			OperatingSystem.CURRENT_OPERATING_SYSTEM == OperatingSystem.WINDOWS);
	}

	@Before
	public void setupTraceRmi() throws Throwable {
		traceRmi = addPlugin(tool, TraceRmiPlugin.class);

		pythonPath = getPathToPython();

		assertTrue(pythonPath.toFile().exists());
		outFile = Files.createTempFile("pydbgout", null);
		errFile = Files.createTempFile("pydbgerr", null);
	}

	protected Path getPathToPython() {
		try {
			String py3path = DummyProc.which("python3");
			if (py3path != null && !py3path.contains("msys")) {
				return Paths.get(py3path);
			}
		}
		catch (RuntimeException e) {
			// Should not happen since we maintain library on host test machine
		}
		return Paths.get(DummyProc.which("python"));
	}

	protected void addAllDebuggerPlugins() throws PluginException {
		PluginsConfiguration plugConf = new PluginsConfiguration() {
			@Override
			protected boolean accepts(Class<? extends Plugin> pluginClass) {
				return !ApplicationLevelOnlyPlugin.class.isAssignableFrom(pluginClass);
			}
		};

		for (PluginDescription pd : plugConf
				.getPluginDescriptions(PluginPackage.getPluginPackage("Debugger"))) {
			addPlugin(tool, pd.getPluginClass());
		}
	}

	protected static String addrToStringForPython(InetAddress address) {
		if (address.isAnyLocalAddress()) {
			return "127.0.0.1"; // Can't connect to 0.0.0.0 as such. Choose localhost.
		}
		return address.getHostAddress();
	}

	protected static String sockToStringForPython(SocketAddress address) {
		if (address instanceof InetSocketAddress tcp) {
			return addrToStringForPython(tcp.getAddress()) + ":" + tcp.getPort();
		}
		throw new AssertionError("Unhandled address type " + address);
	}

	protected record PythonResult(boolean timedOut, int exitCode, String stdout, String stderr) {
		protected String handle() {
			if (stderr.contains("Error") || (0 != exitCode && 1 != exitCode && 143 != exitCode)) {
				throw new PythonError(exitCode, stdout, stderr);
			}
			return stdout;
		}
	}

	protected record ExecInPython(Process python, CompletableFuture<PythonResult> future) {}

	protected void pump(InputStream streamIn, OutputStream streamOut) {
		Thread t = new Thread(() -> {
			try (PrintStream printOut = new PrintStream(streamOut);
					BufferedReader reader = new BufferedReader(new InputStreamReader(streamIn))) {
				String line;
				while ((line = reader.readLine()) != null) {
					printOut.println(line);
					printOut.flush();
				}
			}
			catch (IOException e) {
				Msg.info(this, "Terminating stdin pump, because " + e);
			}
		});
		t.setDaemon(true);
		t.start();
	}

	protected void pumpTee(InputStream streamIn, File fileOut, PrintStream streamOut) {
		Thread t = new Thread(() -> {
			try (PrintStream fileStream = new PrintStream(fileOut);
					BufferedReader reader = new BufferedReader(new InputStreamReader(streamIn))) {
				String line;
				while ((line = reader.readLine()) != null) {
					streamOut.println(line);
					streamOut.flush();
					fileStream.println(line);
					fileStream.flush();
				}
			}
			catch (IOException e) {
				Msg.info(this, "Terminating tee: " + fileOut + ", because " + e);
			}
		});
		t.setDaemon(true);
		t.start();
	}

	@SuppressWarnings("resource") // Do not close stdin 
	protected ExecInPython execInPython(String script) throws IOException {
		ProcessBuilder pb = new ProcessBuilder(pythonPath.toString(), "-i");
		setPythonPath(pb);
		setWindbgPath(pb);

		// If commands come from file, Python will quit after EOF.
		Msg.info(this, "outFile: " + outFile);
		Msg.info(this, "errFile: " + errFile);

		//pb.inheritIO();
		pb.redirectInput(ProcessBuilder.Redirect.PIPE);
		if (SystemUtilities.isInTestingBatchMode()) {
			pb.redirectOutput(outFile.toFile());
			pb.redirectError(errFile.toFile());
		}
		else {
			pb.redirectOutput(ProcessBuilder.Redirect.PIPE);
			pb.redirectError(ProcessBuilder.Redirect.PIPE);
		}
		Process pyproc = pb.start();

		if (!SystemUtilities.isInTestingBatchMode()) {
			pumpTee(pyproc.getInputStream(), outFile.toFile(), System.out);
			pumpTee(pyproc.getErrorStream(), errFile.toFile(), System.err);
		}

		OutputStream stdin = pyproc.getOutputStream();
		stdin.write(script.getBytes());
		stdin.flush();

		if (!SystemUtilities.isInTestingBatchMode()) {
			pump(System.in, stdin);
		}

		return new ExecInPython(pyproc, CompletableFuture.supplyAsync(() -> {
			try {
				if (!pyproc.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
					Msg.error(this, "Timed out waiting for Python");
					pyproc.destroyForcibly();
					pyproc.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS);
					return new PythonResult(true, -1, Files.readString(outFile),
						Files.readString(errFile));
				}
				Msg.info(this, "Python exited with code " + pyproc.exitValue());
				return new PythonResult(false, pyproc.exitValue(), Files.readString(outFile),
					Files.readString(errFile));
			}
			catch (Exception e) {
				return ExceptionUtils.rethrow(e);
			}
			finally {
				pyproc.destroyForcibly();
			}
		}));
	}

	public static class PythonError extends RuntimeException {
		public final int exitCode;
		public final String stdout;
		public final String stderr;

		public PythonError(int exitCode, String stdout, String stderr) {
			super("""
					exitCode=%d:
					----stdout----
					%s
					----stderr----
					%s
					""".formatted(exitCode, stdout, stderr));
			this.exitCode = exitCode;
			this.stdout = stdout;
			this.stderr = stderr;
		}
	}

	protected String runThrowError(String script) throws Exception {
		CompletableFuture<PythonResult> result = execInPython(script).future;
		return result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
	}

	protected record PythonAndConnection(ExecInPython exec, TraceRmiConnection connection)
			implements AutoCloseable {
		protected RemoteMethod getMethod(String name) {
			return Objects.requireNonNull(connection.getMethods().get(name));
		}

		public void execute(String cmd) {
			RemoteMethod execute = getMethod("execute");
			execute.invoke(Map.of("cmd", cmd));
		}

		public RemoteAsyncResult executeAsync(String cmd) {
			RemoteMethod execute = getMethod("execute");
			return execute.invokeAsync(Map.of("cmd", cmd));
		}

		public String executeCapture(String cmd) {
			RemoteMethod execute = getMethod("execute");
			return (String) execute.invoke(Map.of("cmd", cmd, "to_string", true));
		}

		@Override
		public void close() throws Exception {
			Msg.info(this, "Cleaning up python");
			exec.python().destroy();
			try {
				PythonResult r = exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
				r.handle();
				waitForPass(this, () -> assertTrue(connection.isClosed()),
					TIMEOUT_SECONDS, TimeUnit.SECONDS);
			}
			finally {
				exec.python.destroyForcibly();
			}
		}
	}

	protected PythonAndConnection startAndConnectPython(Function<String, String> scriptSupplier)
			throws Exception {
		TraceRmiAcceptor acceptor = traceRmi.acceptOne(null);
		ExecInPython exec =
			execInPython(scriptSupplier.apply(sockToStringForPython(acceptor.getAddress())));
		acceptor.setTimeout(CONNECT_TIMEOUT_MS);
		try {
			TraceRmiConnection connection = acceptor.accept();
			return new PythonAndConnection(exec, connection);
		}
		catch (SocketTimeoutException e) {
			exec.python.destroyForcibly();
			exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
			throw e;
		}
	}

	protected PythonAndConnection startAndConnectPython() throws Exception {
		return startAndConnectPython(addr -> """
				%s
				ghidra_trace_connect('%s')
				""".formatted(PREAMBLE, addr));
	}

	@SuppressWarnings("resource")
	protected String runThrowError(Function<String, String> scriptSupplier)
			throws Exception {
		PythonAndConnection conn = startAndConnectPython(scriptSupplier);
		PythonResult r = conn.exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
		String stdout = r.handle();
		waitForPass(this, () -> assertTrue(conn.connection.isClosed()),
			TIMEOUT_SECONDS, TimeUnit.SECONDS);
		return stdout;
	}

	protected void waitStopped(String message) {
		TraceObject proc =
			Objects.requireNonNull(tb.objAny("Sessions[].Processes[]", Lifespan.at(0)));
		waitForPass(() -> assertEquals(message, "STOPPED", tb.objValue(proc, 0, "_state")));
		waitTxDone();
	}

	protected void waitRunning(String message) {
		TraceObject proc =
			Objects.requireNonNull(tb.objAny("Sessions[].Processes[]", Lifespan.at(0)));
		waitForPass(() -> assertEquals(message, "RUNNING", tb.objValue(proc, 0, "_state")));
		waitTxDone();
	}

	protected String extractOutSection(String out, String head) {
		String[] split = out.split("\n");
		String xout = "";
		for (String s : split) {
			if (!s.startsWith("(python)") && !s.equals("")) {
				xout += s + "\n";
			}
		}
		return xout.split(head)[1].split("---")[0].replace("(python)", "").trim();
	}

	record MemDump(long address, byte[] data) {}

	protected MemDump parseHexDump(String dump) throws IOException {
		// First, get the address. Assume contiguous, so only need top line.
		List<String> lines = List.of(dump.split("\n"));
		List<String> toksLine0 = List.of(lines.get(0).split("\\s+"));
		String addrstr = toksLine0.get(0);
		if (addrstr.contains(":")) {
			addrstr = addrstr.substring(0, addrstr.indexOf(":"));
		}
		long address = Long.parseLong(addrstr, 16);

		ByteArrayOutputStream buf = new ByteArrayOutputStream();
		for (String l : lines) {
			List<String> parts = List.of(l.split(":"));
			assertEquals(2, parts.size());
			String hex = parts.get(1).substring(0, 48);
			byte[] lineData = NumericUtilities.convertStringToBytes(hex);
			assertNotNull("Converted to null: " + hex, parts.get(1));
			buf.write(lineData);
		}
		return new MemDump(address, buf.toByteArray());
	}

	record RegDump() {}

	protected RegDump parseRegDump(String dump) {
		return new RegDump();
	}

	protected ManagedDomainObject<Trace> openTrace(String path) throws Exception {
		DomainFile df = env.getProject().getProjectData().getFile(path);
		assertNotNull(df);
		return new ManagedDomainObject<>(df, Trace.class, monitor);
	}

	protected ManagedDomainObject<Trace> waitTrace(String path) throws Exception {
		DomainFile df;
		long start = System.currentTimeMillis();
		while (true) {
			df = env.getProject().getProjectData().getFile(path);
			if (df != null) {
				return new ManagedDomainObject<>(df, Trace.class, monitor);
			}
			Thread.sleep(1000);
			if (System.currentTimeMillis() - start > 30000) {
				throw new TimeoutException("30 seconds expired waiting for domain file");
			}
		}
	}

	protected void assertBreakLoc(TraceObjectValue locVal, String key, Address addr, int len,
			Set<TraceBreakpointKind> kinds, String expression) throws Exception {
		assertEquals(key, locVal.getEntryKey());
		TraceObject loc = locVal.getChild();
		TraceObject spec = loc;
		assertEquals(new AddressRangeImpl(addr, len), loc.getValue(0, "_range").getValue());
		assertEquals(TraceBreakpointKindSet.encode(kinds), spec.getValue(0, "_kinds").getValue());
		assertTrue(spec.getValue(0, "_expression").getValue().toString().contains(expression));
	}

	protected void assertWatchLoc(TraceObjectValue locVal, String key, Address addr, int len,
			Set<TraceBreakpointKind> kinds, String expression) throws Exception {
		assertEquals(key, locVal.getEntryKey());
		TraceObject loc = locVal.getChild();
		assertEquals(new AddressRangeImpl(addr, len), loc.getValue(0, "_range").getValue());
		assertEquals(TraceBreakpointKindSet.encode(kinds), loc.getValue(0, "_kinds").getValue());
	}

	protected void waitTxDone() {
		waitFor(() -> tb.trace.getCurrentTransactionInfo() == null);
	}

	public static void waitForPass(Runnable runnable, long timeoutMs, long retryDelayMs) {
		long start = System.currentTimeMillis();
		AssertionError lastError = null;
		while (System.currentTimeMillis() - start < timeoutMs) {
			try {
				runnable.run();
				return;
			}
			catch (AssertionError e) {
				lastError = e;
			}
			try {
				Thread.sleep(retryDelayMs);
			}
			catch (InterruptedException e) {
				// Retry sooner, I guess.
			}
		}
		if (lastError == null) {
			throw new AssertionError("Timed out before first try?");
		}
		throw lastError;
	}
}