Skip to content

Ghidra is a software reverse engineering (SRE) framework

21.4 KB
Raw Blame
/* ###
 * IP: GHIDRA
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package agent.lldb.rmi;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;

import java.io.*;
import java.net.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.*;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.apache.commons.io.output.TeeOutputStream;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.hamcrest.Matchers;
import org.junit.*;

import ghidra.app.plugin.core.debug.gui.AbstractGhidraHeadedDebuggerTest;
import ghidra.app.plugin.core.debug.service.tracermi.TraceRmiPlugin;
import ghidra.app.plugin.core.debug.utils.ManagedDomainObject;
import ghidra.app.services.TraceRmiService;
import ghidra.debug.api.tracermi.*;
import ghidra.framework.*;
import ghidra.framework.main.ApplicationLevelOnlyPlugin;
import ghidra.framework.model.DomainFile;
import ghidra.framework.plugintool.Plugin;
import ghidra.framework.plugintool.PluginsConfiguration;
import ghidra.framework.plugintool.util.*;
import ghidra.program.model.address.Address;
import ghidra.program.model.address.AddressRangeImpl;
import ghidra.pty.*;
import ghidra.pty.testutil.DummyProc;
import ghidra.pty.windows.AnsiBufferedInputStream;
import ghidra.trace.model.Lifespan;
import ghidra.trace.model.Trace;
import ghidra.trace.model.breakpoint.TraceBreakpointKind;
import ghidra.trace.model.breakpoint.TraceBreakpointKind.TraceBreakpointKindSet;
import ghidra.trace.model.target.TraceObject;
import ghidra.trace.model.target.TraceObjectValue;
import ghidra.util.*;

public abstract class AbstractLldbTraceRmiTest extends AbstractGhidraHeadedDebuggerTest {

	public static final boolean IS_WINDOWS =
		OperatingSystem.CURRENT_OPERATING_SYSTEM == OperatingSystem.WINDOWS;

	record PlatDep(String name, String endian, String lang, String cSpec, String callMne,
			String intReg, String floatReg) {
		static final PlatDep ARM64 =
			new PlatDep("arm64", "little", "AARCH64:LE:64:v8A", "default", "bl", "x0", "s0");
		static final PlatDep X8664 = // Note AT&T callq
			new PlatDep("x86_64", "little", "x86:LE:64:default", IS_WINDOWS ? "windows" : "gcc",
				"callq", "rax", "st0");
	}

	public static final PlatDep PLAT = computePlat();

	protected static boolean didSetupPython = false;

	static PlatDep computePlat() {
		return switch (System.getProperty("os.arch")) {
			case "aarch64" -> PlatDep.ARM64;
			case "x86" -> PlatDep.X8664;
			case "amd64" -> PlatDep.X8664;
			default -> throw new AssertionError(
				"Unrecognized arch: " + System.getProperty("os.arch"));
		};
	}

	static String getSpecimenNewThreadAndExit() {
		return IS_WINDOWS ? which("expCreateThreadExit") : which("expCloneExit");
	}

	static int getSleepThreadCount() {
		// The targets above use different sleep methods:
		//   Linux spawns clock_nanosleep
		//   Windows spawns ZwDelayExecution and multiple ZwWaitForWork threads
		return IS_WINDOWS ? 2 : 1;
	}

	static String getSpecimenPrint() {
		return which("expPrint");
	}

	static String getSpecimenSpin() {
		return which("expSpin");
	}

	static String getSpecimenRead() {
		return which("expRead");
	}

	/**
	 * Some features have to be disabled to avoid permissions issues in the test container. Namely,
	 * don't try to disable ASLR.
	 * 
	 * Color codes mess up the address parsing.
	 */
	public static final String PREAMBLE = """
			script import ghidralldb
			settings set use-color false
			settings set target.disable-aslr false
			""";
	// Connecting should be the first thing the script does, so use a tight timeout.
	protected static final int CONNECT_TIMEOUT_MS = 3000;
	protected static final int TIMEOUT_SECONDS = SystemUtilities.isInTestingBatchMode() ? 10 : 300;
	protected static final int QUIT_TIMEOUT_MS = 1000;

	/** Some snapshot likely to exceed the latest */
	protected static final long SNAP = 100;

	protected TraceRmiService traceRmi;
	private Path lldbPath;

	@BeforeClass
	public static void setupPython() throws Throwable {
		if (didSetupPython) {
			// Only do this once when running the full suite.
			return;
		}
		if (SystemUtilities.isInTestingBatchMode()) {
			// Don't run gradle in gradle. It already did this task.
			return;
		}
		String gradleCmd =
			OperatingSystem.CURRENT_OPERATING_SYSTEM == OperatingSystem.WINDOWS ? "gradle.bat"
					: "gradle";
		String gradle = DummyProc.which(gradleCmd);
		new ProcessBuilder(gradle, "assemblePyPackage")
				.directory(TestApplicationUtils.getInstallationDirectory())
				.inheritIO()
				.start()
				.waitFor();
		didSetupPython = true;
	}

	protected void setPythonPath(Map<String, String> env) throws IOException {
		String sep =
			OperatingSystem.CURRENT_OPERATING_SYSTEM == OperatingSystem.WINDOWS ? ";" : ":";
		String rmiPyPkg = Application.getModuleSubDirectory("Debugger-rmi-trace",
			"build/pypkg/src").getAbsolutePath();
		String gdbPyPkg = Application.getModuleSubDirectory("Debugger-agent-lldb",
			"build/pypkg/src").getAbsolutePath();
		String add = rmiPyPkg + sep + gdbPyPkg;
		env.compute("PYTHONPATH", (k, v) -> v == null ? add : (v + sep + add));
	}

	@Before
	public void setupTraceRmi() throws Throwable {
		traceRmi = addPlugin(tool, TraceRmiPlugin.class);

		traceManager.setSaveTracesByDefault(false);

		try {
			lldbPath = Paths.get(DummyProc.which("lldb-20"));
		}
		catch (RuntimeException e) {
			lldbPath = Paths.get(DummyProc.which("lldb"));
		}
	}

	@After
	public void tearDownTraceRmi() throws IOException {
		for (TraceRmiConnection cx : traceRmi.getAllConnections()) {
			cx.close();
		}
	}

	protected void addAllDebuggerPlugins() throws PluginException {
		PluginsConfiguration plugConf = new PluginsConfiguration() {
			@Override
			protected boolean accepts(Class<? extends Plugin> pluginClass) {
				return !ApplicationLevelOnlyPlugin.class.isAssignableFrom(pluginClass);
			}
		};

		for (PluginDescription pd : plugConf
				.getPluginDescriptions(PluginPackage.getPluginPackage("Debugger"))) {
			addPlugin(tool, pd.getPluginClass());
		}
	}

	protected static String addrToStringForLldb(InetAddress address) {
		if (address.isAnyLocalAddress()) {
			return "127.0.0.1"; // Can't connect to 0.0.0.0 as such. Choose localhost.
		}
		return address.getHostAddress();
	}

	protected static String sockToStringForLldb(SocketAddress address) {
		if (address instanceof InetSocketAddress tcp) {
			return addrToStringForLldb(tcp.getAddress()) + ":" + tcp.getPort();
		}
		throw new AssertionError("Unhandled address type " + address);
	}

	protected record LldbResult(boolean timedOut, int exitCode, String out) {
		protected String handle() {
			if (0 != exitCode && 143 != exitCode) {
				throw new LldbError(exitCode, out);
			}
			return out;
		}
	}

	protected record ExecInLldb(Pty pty, PtySession lldb, CompletableFuture<LldbResult> future,
			Thread pumper, ByteArrayOutputStream capture) {}

	@SuppressWarnings("resource") // Do not close stdin 
	protected ExecInLldb execInLldb(String script) throws IOException {
		script = lfIfWindows(script);
		Pty pty = PtyFactory.local().openpty();
		Map<String, String> env = new HashMap<>(System.getenv());
		setPythonPath(env);
		env.put("TERM", "xterm-256color");
		ByteArrayOutputStream capture = new ByteArrayOutputStream();
		OutputStream out;
		if (SystemUtilities.isInTestingBatchMode()) {
			out = capture;
		}
		else {
			out = new TeeOutputStream(System.out, capture);
		}
		InputStream inputStream = pty.getParent().getInputStream();
		inputStream = new AnsiBufferedInputStream(inputStream);
		Thread pumper = new StreamPumper(inputStream, out);
		pumper.start();
		// NB: you have to do this because the AnsiBufferedInputStream requires a single line
		pty.getChild().setWindowSize((short) 400, (short) 1);
		PtySession lldbSession = pty.getChild().session(new String[] { lldbPath.toString() }, env);

		OutputStream stdin = pty.getParent().getOutputStream();
		stdin.write(script.getBytes());
		stdin.flush();
		return new ExecInLldb(pty, lldbSession, CompletableFuture.supplyAsync(() -> {
			try {
				int exitVal = lldbSession.waitExited(TIMEOUT_SECONDS, TimeUnit.SECONDS);
				Msg.info(this, "LLDB exited with code " + exitVal);
				return new LldbResult(false, exitVal, capture.toString());
			}
			catch (TimeoutException e) {
				return new LldbResult(true, 0, capture.toString());
			}
			catch (Exception e) {
				return ExceptionUtils.rethrow(e);
			}
			finally {
				try {
					pty.close();
				}
				catch (IOException e) {
					Msg.warn(this, "Couldn't close pty: " + e);
				}
				lldbSession.destroyForcibly();
				pumper.interrupt();
			}
		}), pumper, capture);
	}

	public static class LldbError extends RuntimeException {
		public final int exitCode;
		public final String out;

		public LldbError(int exitCode, String out) {
			super("""
					exitCode=%d:
					----out----
					%s
					""".formatted(exitCode, out));
			this.exitCode = exitCode;
			this.out = out;
		}
	}

	protected String runThrowError(String script) throws Exception {
		CompletableFuture<LldbResult> result = execInLldb(script).future;
		return result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
	}

	protected class LldbAndConnection implements AutoCloseable {
		private final ExecInLldb exec;
		private final TraceRmiConnection connection;
		private boolean success = false;

		public LldbAndConnection(ExecInLldb exec, TraceRmiConnection connection) {
			this.exec = exec;
			this.connection = connection;
		}

		public TraceRmiConnection connection() {
			return connection;
		}

		protected RemoteMethod getMethod(String name) {
			return Objects.requireNonNull(connection.getMethods().get(name));
		}

		public void execute(String cmd) {
			cmd = lfIfWindows(cmd);
			RemoteMethod execute = getMethod("execute");
			execute.invoke(Map.of("cmd", cmd));
		}

		public RemoteAsyncResult executeAsync(String cmd) {
			cmd = lfIfWindows(cmd);
			RemoteMethod execute = getMethod("execute");
			return execute.invokeAsync(Map.of("cmd", cmd));
		}

		public String executeCapture(String cmd) {
			cmd = lfIfWindows(cmd);
			RemoteMethod execute = getMethod("execute");
			return (String) execute.invoke(Map.of("cmd", cmd, "to_string", true));
		}

		public Object evaluate(String expr) {
			expr = lfIfWindows(expr);
			RemoteMethod evaluate = getMethod("evaluate");
			return evaluate.invoke(Map.of("expr", expr));
		}

		public Object pyeval(String expr) {
			RemoteMethod pyeval = getMethod("pyeval");
			return pyeval.invoke(Map.of("expr", expr));
		}

		public void success() {
			success = true;
		}

		@Override
		public void close() throws Exception {
			Msg.info(this, "Cleaning up lldb");
			execute("settings set auto-confirm true");
			String cmd = """
					quit
					""";
			cmd = lfIfWindows(cmd);
			exec.pty.getParent().getOutputStream().write(cmd.getBytes());
			try {
				try {
					LldbResult r = exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
					r.handle();
				}
				catch (Exception e) {
					Msg.error(this, e);
				}
				waitForPass(this, () -> assertTrue(connection.isClosed()), TIMEOUT_SECONDS,
					TimeUnit.SECONDS);
			}
			finally {
				if (!success) {
					Msg.info(this, "LLDB output:\n" + exec.capture.toString());
				}

				exec.pty.close();
				exec.lldb.destroyForcibly();
				exec.pumper.interrupt();
			}
		}
	}

	protected LldbAndConnection startAndConnectLldb(Function<String, String> scriptSupplier)
			throws Exception {
		TraceRmiAcceptor acceptor = traceRmi.acceptOne(null);
		ExecInLldb exec =
			execInLldb(scriptSupplier.apply(sockToStringForLldb(acceptor.getAddress())));
		acceptor.setTimeout(CONNECT_TIMEOUT_MS);
		try {
			TraceRmiConnection connection = acceptor.accept();
			return new LldbAndConnection(exec, connection);
		}
		catch (SocketTimeoutException e) {
			exec.pty.close();
			exec.lldb.destroyForcibly();
			exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
			exec.pumper.interrupt();
			throw e;
		}
	}

	protected LldbAndConnection startAndConnectLldb() throws Exception {
		return startAndConnectLldb(addr -> """
				%s
				ghidra trace connect %s
				""".formatted(PREAMBLE, addr));
	}

	@SuppressWarnings("resource")
	protected String runThrowError(Function<String, String> scriptSupplier)
			throws Exception {
		LldbAndConnection conn = startAndConnectLldb(scriptSupplier);
		LldbResult r = conn.exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
		conn.exec.pty.close();
		conn.exec.pumper.interrupt();
		String stdout = r.handle();
		waitForPass(() -> assertTrue(conn.connection.isClosed()));
		return stdout;
	}

	protected void waitState(String state) {
		TraceObject proc = Objects.requireNonNull(tb.objAny("Processes[]", Lifespan.at(0)));
		for (int i = 0; i < 5; i++) {
			try {
				waitForPass(() -> {
					Long snap = tb.trace.getTimeManager().getMaxSnap();
					assertEquals(state, tb.objValue(proc, snap != null ? snap : 0, "_state"));
				});
				break;
			}
			catch (AssertionError e) {
				if (i == 4) {
					throw e;
				}
			}
		}
		waitTxDone();
	}

	protected void waitStopped(LldbAndConnection conn) {
		waitForPass(() -> assertEquals(Boolean.TRUE,
			conn.pyeval("util.get_debugger().GetTargetAtIndex(0).GetProcess().is_stopped")));
		// waitState("STOPPED");
	}

	protected void waitRunning(LldbAndConnection conn) {
		waitForPass(() -> assertEquals(Boolean.TRUE,
			conn.pyeval("util.get_debugger().GetTargetAtIndex(0).GetProcess().is_running")));
		// waitState("RUNNING");
	}

	protected String extractOutSection(String out, String head) {
		String[] split = out.replace("\r", "").split("\n");
		String xout = "";
		for (String s : split) {
			if (!s.contains("script print(") && !s.equals("")) {
				xout += s + "\n";
			}
		}
		return xout.split(head)[1].split("---")[0].replace("(lldb)", "").trim();
	}

	// OK, Windows versions just behave differently re prompt
	protected String extractOutSectionWithPrompt(String out, String head) {
		String[] split = out.replace("\r", "").split("\n");
		String xout = "";
		for (String s : split) {
			if (!s.contains("script print(") && !s.equals("")) {
				xout += s + "\n";
			}
		}
		return xout.split(head)[1].split("---")[0].trim();
	}

	record MemDump(long address, byte[] data) {}

	protected MemDump parseHexDump(String dump) throws IOException {
		// First, get the address. Assume contiguous, so only need top line.
		List<String> lines = List.of(dump.split("\n"));
		List<String> toksLine0 = List.of(lines.get(0).split("\\s+"));
		assertThat(toksLine0.get(0), startsWith("0x"));
		String addrstr = toksLine0.get(0);
		if (addrstr.contains(":")) {
			addrstr = addrstr.substring(0, addrstr.indexOf(":"));
		}
		long address = Long.decode(addrstr);

		ByteArrayOutputStream buf = new ByteArrayOutputStream();
		for (String l : lines) {
			List<String> parts = List.of(l.split(":"));
			assertEquals(2, parts.size());
			String hex = parts.get(1).replaceAll("\\s*0x", "");
			byte[] lineData = NumericUtilities.convertStringToBytes(hex);
			assertNotNull("Converted to null: " + hex, parts.get(1));
			buf.write(lineData);
		}
		return new MemDump(address, buf.toByteArray());
	}

	protected void waitDomainObjectClosed(String path) {
		DomainFile df = env.getProject().getProjectData().getFile(path);
		assertNotNull(df);
		waitForPass(() -> assertFalse(df.isOpen()));
	}

	protected ManagedDomainObject<Trace> openTrace(String path) throws Exception {
		DomainFile df = env.getProject().getProjectData().getFile(path);
		assertNotNull(df);
		return new ManagedDomainObject<>(df, Trace.class, monitor);
	}

	protected ManagedDomainObject<Trace> waitTrace(String path) throws Exception {
		DomainFile df;
		long start = System.currentTimeMillis();
		while (true) {
			df = env.getProject().getProjectData().getFile(path);
			if (df != null) {
				return new ManagedDomainObject<>(df, Trace.class, monitor);
			}
			Thread.sleep(1000);
			if (System.currentTimeMillis() - start > 30000) {
				throw new TimeoutException("30 seconds expired waiting for domain file");
			}
		}
	}

	protected void assertLocalOs(String actual) {
		assertThat(actual, Matchers.startsWith(switch (OperatingSystem.CURRENT_OPERATING_SYSTEM) {
			case LINUX -> "linux";
			case WINDOWS -> "windows";
			case MAC_OS_X -> "macos";
			default -> throw new AssertionError("What OS?");
		}));
	}

	protected void assertBreakLoc(TraceObjectValue locVal, String key, Address addr, int len,
			Set<TraceBreakpointKind> kinds, String expression) throws Exception {
		assertEquals(key, locVal.getEntryKey());
		TraceObject loc = locVal.getChild();
		TraceObject spec = loc.getCanonicalParent(0).getParent();
		assertEquals(new AddressRangeImpl(addr, len), loc.getValue(0, "_range").getValue());
		assertEquals(TraceBreakpointKindSet.encode(kinds), spec.getValue(0, "_kinds").getValue());
		assertTrue(spec.getValue(0, "_expression").getValue().toString().contains(expression));
	}

	protected void assertWatchLoc(TraceObjectValue locVal, String key, Address addr, int len,
			Set<TraceBreakpointKind> kinds, String expression) throws Exception {
		assertEquals(key, locVal.getEntryKey());
		TraceObject loc = locVal.getChild();
		assertEquals(new AddressRangeImpl(addr, len), loc.getValue(0, "_range").getValue());
		assertEquals(TraceBreakpointKindSet.encode(kinds), loc.getValue(0, "_kinds").getValue());
	}

	protected void waitTxDone() {
		waitFor(() -> tb.trace.getCurrentTransactionInfo() == null);
	}

	private record Cut(String head, int begin, int end) {
		String parseCell(String line) {
			int begin = Math.min(line.length(), this.begin);
			int end = Math.min(line.length(), this.end);
			/**
			 * NOTE: Do not assert previous char is space.
			 * 
			 * When breakpoints table spells out locations, Address and What cells are indented and
			 * no longer align with their column headers.
			 */
			return line.substring(begin, end).trim();
		}
	}

	protected record Row(Map<String, String> cells) {
		private static Row parse(List<Cut> cuts, String line) {
			return new Row(
				cuts.stream().collect(Collectors.toMap(Cut::head, c -> c.parseCell(line))));
		}

		public String getCell(String head) {
			return cells.get(head);
		}
	}

	protected record Tabular(List<String> headings, List<Row> rows) {

		static final Pattern SPACES = Pattern.compile(" *");
		static final Pattern WORDS = Pattern.compile("\\w+");

		private static List<Cut> findCuts(String header) {
			List<Cut> result = new ArrayList<>();
			Matcher spaceMatcher = SPACES.matcher(header);
			Matcher wordMatcher = WORDS.matcher(header);
			int start = 0;
			while (start < header.length()) {
				if (!spaceMatcher.find(start)) {
					throw new AssertionError();
				}
				start = spaceMatcher.end();
				if (start >= header.length()) {
					break;
				}
				if (!wordMatcher.find(start)) {
					throw new AssertionError();
				}
				result.add(new Cut(wordMatcher.group(), wordMatcher.start(), wordMatcher.end()));
				start = wordMatcher.end();
			}
			return result;
		}

		private static List<Cut> adjustCuts(List<Cut> cuts) {
			List<Cut> result = new ArrayList<>();
			for (int i = 0; i < cuts.size(); i++) {
				Cut cut = cuts.get(i);
				int j = i + 1;
				int end = j < cuts.size() ? cuts.get(j).begin : Integer.MAX_VALUE;
				result.add(new Cut(cut.head, cut.begin, end));
			}
			return result;
		}

		/**
		 * Parse a table.
		 * 
		 * <p>
		 * This is far from perfect, but good enough for making assertions in tests. For example, in
		 * the breakpoints table, lldb may insert an extra informational line under a breakpoint
		 * row. This line will get mangled and parsed as if it were an entry. However, it's "Num"
		 * cell will be empty, so they will not likely interfere.
		 * 
		 * @param out the output in tabular form
		 * @return the table object, more or less
		 */
		public static Tabular parse(String out) {
			List<String> lines = List.of(out.split("\n"));
			if (lines.isEmpty()) {
				throw new AssertionError("Output is not tabular");
			}
			List<Cut> cuts = adjustCuts(findCuts(lines.get(0)));
			return new Tabular(cuts.stream().map(Cut::head).toList(),
				lines.stream().skip(1).map(l -> Row.parse(cuts, l)).toList());
		}

		public Row findRow(String head, String contents) {
			return rows.stream()
					.filter(r -> Objects.equals(contents, r.getCell(head)))
					.findFirst()
					.orElse(null);
		}
	}

	public static void waitForPass(Runnable runnable, long timeoutMs, long retryDelayMs) {
		long start = System.currentTimeMillis();
		AssertionError lastError = null;
		while (System.currentTimeMillis() - start < timeoutMs) {
			try {
				runnable.run();
				return;
			}
			catch (AssertionError e) {
				lastError = e;
			}
			try {
				Thread.sleep(retryDelayMs);
			}
			catch (InterruptedException e) {
				// Retry sooner, I guess.
			}
		}
		if (lastError == null) {
			throw new AssertionError("Timed out before first try?");
		}
		throw lastError;
	}

	public static String which(String cmd) {
		return DummyProc.which(cmd).replace('\\', '/');
	}

	public static String projectName(String cmd) {
		String ext = IS_WINDOWS ? ".exe" : "";
		return "/New Traces/lldb/" + cmd + ext;
	}

	private String lfIfWindows(String cmd) {
		if (IS_WINDOWS) {
			cmd = cmd.replace("\n", "\r\n");
		}
		return cmd;
	}
}