Skip to content

Ghidra is a software reverse engineering (SRE) framework

18.8 KB
Raw Blame
/* ###
 * IP: GHIDRA
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package agent;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.*;

import java.io.*;
import java.net.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.*;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.exception.ExceptionUtils;
import org.hamcrest.Matchers;
import org.junit.*;

import db.NoTransactionException;
import generic.Unique;
import ghidra.app.plugin.core.debug.gui.AbstractGhidraHeadedDebuggerTest;
import ghidra.app.plugin.core.debug.service.tracermi.TraceRmiPlugin;
import ghidra.app.plugin.core.debug.utils.ManagedDomainObject;
import ghidra.app.services.TraceRmiService;
import ghidra.debug.api.tracermi.*;
import ghidra.framework.*;
import ghidra.framework.main.ApplicationLevelOnlyPlugin;
import ghidra.framework.model.DomainFile;
import ghidra.framework.plugintool.Plugin;
import ghidra.framework.plugintool.PluginsConfiguration;
import ghidra.framework.plugintool.util.*;
import ghidra.pty.*;
import ghidra.pty.PtyChild.Echo;
import ghidra.pty.testutil.DummyProc;
import ghidra.pty.windows.AnsiBufferedInputStream;
import ghidra.trace.model.Trace;
import ghidra.trace.model.target.schema.PrimitiveTraceObjectSchema.MinimalSchemaContext;
import ghidra.trace.model.target.schema.TraceObjectSchema.SchemaName;
import ghidra.trace.model.time.TraceSnapshot;
import ghidra.util.Msg;
import ghidra.util.SystemUtilities;

public class TraceRmiPythonClientTest extends AbstractGhidraHeadedDebuggerTest {
	public static final String PREAMBLE =
		"""
				import socket
				from typing import Annotated, Any, Optional

				from concurrent.futures import ThreadPoolExecutor
				from ghidratrace import sch
				from ghidratrace.client import (Client, Address, AddressRange, TraceObject,
				    MethodRegistry, Schedule, TraceRmiError, ParamDesc)

				registry = MethodRegistry(ThreadPoolExecutor())

				def connect(addr):
				    cs = socket.socket()
				    cs.connect(addr)
				    return Client(cs, "test-client", registry)

				""";
	protected static final int CONNECT_TIMEOUT_MS = 3000;
	protected static final int TIMEOUT_SECONDS = 10;
	protected static final int QUIT_TIMEOUT_MS = 1000;

	protected TraceRmiService traceRmi;
	private Path pathToPython;

	@BeforeClass
	public static void setupPython() throws Throwable {
		if (SystemUtilities.isInTestingBatchMode()) {
			return; // gradle should already have done it
		}
		String gradle = switch (OperatingSystem.CURRENT_OPERATING_SYSTEM) {
			case WINDOWS -> DummyProc.which("gradle.bat");
			default -> "gradle";
		};
		assertEquals(0, new ProcessBuilder(gradle, "Debugger-rmi-trace:assemblePyPackage")
				.directory(TestApplicationUtils.getInstallationDirectory())
				.inheritIO()
				.start()
				.waitFor());
	}

	protected void setPythonPath(Map<String, String> env) throws IOException {
		String sep =
			OperatingSystem.CURRENT_OPERATING_SYSTEM == OperatingSystem.WINDOWS ? ";" : ":";
		String rmiPyPkg = Application.getModuleSubDirectory("Debugger-rmi-trace",
			"build/pypkg/src").getAbsolutePath();
		String add = rmiPyPkg;
		env.compute("PYTHONPATH", (k, v) -> v == null ? add : (v + sep + add));
	}

	protected Path getPathToPython() {
		try {
			String py3path = DummyProc.which("python3");
			if (py3path != null && !py3path.contains("msys")) {
				return Paths.get(py3path);
			}
		}
		catch (RuntimeException e) {
		}
		return Paths.get(DummyProc.which("python"));
	}

	@Before
	public void setupTraceRmi() throws Throwable {
		traceRmi = addPlugin(tool, TraceRmiPlugin.class);

		pathToPython = getPathToPython();
		Msg.info(this, "Using python: %s".formatted(pathToPython));
	}

	protected void addAllDebuggerPlugins() throws PluginException {
		PluginsConfiguration plugConf = new PluginsConfiguration() {
			@Override
			protected boolean accepts(Class<? extends Plugin> pluginClass) {
				return !ApplicationLevelOnlyPlugin.class.isAssignableFrom(pluginClass);
			}
		};

		for (PluginDescription pd : plugConf
				.getPluginDescriptions(PluginPackage.getPluginPackage("Debugger"))) {
			addPlugin(tool, pd.getPluginClass());
		}
	}

	protected static String addrToStringForPython(InetAddress address) {
		if (address.isAnyLocalAddress()) {
			return "127.0.0.1"; // Can't connect to 0.0.0.0 as such. Choose localhost.
		}
		return address.getHostAddress();
	}

	protected static String sockToStringForPython(SocketAddress address) {
		if (address instanceof InetSocketAddress tcp) {
			return "('%s', %d)".formatted(addrToStringForPython(tcp.getAddress()), tcp.getPort());
		}
		throw new AssertionError("Unhandled address type " + address);
	}

	protected static class PyError extends RuntimeException {
		public final int exitCode;
		public final String out;

		public PyError(int exitCode, String out) {
			super("""
					exitCode=%d:
					----out----
					%s
					""".formatted(exitCode, out));
			this.exitCode = exitCode;
			this.out = out;
		}
	}

	protected record PyResult(boolean timedOut, int exitCode, String out) {
		protected String handle() {
			if (0 != exitCode || out.contains("Traceback")) {
				throw new PyError(exitCode, out);
			}
			return out;
		}
	}

	protected record ExecInPy(PtySession session, PrintWriter stdin,
			CompletableFuture<PyResult> future) {}

	@SuppressWarnings("resource") // Do not close stdin 
	protected ExecInPy execInPy(String script) throws IOException {
		Map<String, String> env = new HashMap<>(System.getenv());
		setPythonPath(env);
		/**
		 * A new REPL was instroduced in Python 3.13. Unfortunately, the REPL is in play when we use
		 * a PTY, because it assumes that is a human. It will automatically insert indentation after
		 * pressing ENTER, which really goofs up our input. (It's worth noting, this happens even
		 * when copy-pasting a code block from notepad, which seems like a bug on their part.) This
		 * environment variable (at least for the moment) disables that new REPL.
		 */
		env.put("PYTHON_BASIC_REPL", "1");
		Pty pty = PtyFactory.local().openpty();

		PtySession session =
			pty.getChild().session(new String[] { pathToPython.toString() }, env, Echo.ON);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		new Thread(() -> {
			InputStream is = new AnsiBufferedInputStream(pty.getParent().getInputStream());
			byte[] buf = new byte[1024];
			while (true) {
				try {
					int len = is.read(buf);
					out.write(buf, 0, len);
					System.out.write(buf, 0, len);
				}
				catch (IOException e) {
					System.out.println("<EOF>");
					return;
				}
			}
		}).start();

		PrintWriter stdin = new PrintWriter(pty.getParent().getOutputStream());
		/**
		 * Because we're using a pty, we need to use CR instead of LF, i.e., to simulate the user
		 * pressing ENTER.
		 */
		script = script.replace("\n", "\r");
		stdin.write(script);
		stdin.flush();
		return new ExecInPy(session, stdin, CompletableFuture.supplyAsync(() -> {
			try {
				int exitCode = session.waitExited(TIMEOUT_SECONDS, TimeUnit.SECONDS);
				Msg.info(this, "Python exited with code " + exitCode);
				return new PyResult(false, exitCode, out.toString());
			}
			catch (TimeoutException e) {
				Msg.error(this, "Timed out waiting for GDB");
				session.destroyForcibly();
				try {
					session.waitExited(TIMEOUT_SECONDS, TimeUnit.SECONDS);
				}
				catch (InterruptedException | TimeoutException e1) {
					throw new AssertionError(e1);
				}
				return new PyResult(true, -1, out.toString());
			}
			catch (Exception e) {
				return ExceptionUtils.rethrow(e);
			}
			finally {
				session.destroyForcibly();
			}
		}));
	}

	protected String runThrowError(String script) throws Exception {
		CompletableFuture<PyResult> result = execInPy(script).future;
		return result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
	}

	protected record PyAndConnection(ExecInPy exec, TraceRmiConnection connection)
			implements AutoCloseable {

		protected RemoteMethod getMethod(String name) {
			return Objects.requireNonNull(connection.getMethods().get(name));
		}

		@Override
		public void close() throws Exception {
			Msg.info(this, "Cleaning up python");
			try {
				exec.stdin.println("exit()");
				exec.stdin.close();
				PyResult r = exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
				r.handle();
				waitForPass(() -> assertTrue(connection.isClosed()));
			}
			finally {
				exec.stdin.close();
				exec.session.destroyForcibly();
			}
		}
	}

	protected PyAndConnection startAndConnectPy(Function<String, String> scriptSupplier)
			throws Exception {
		TraceRmiAcceptor acceptor = traceRmi.acceptOne(null);
		ExecInPy exec =
			execInPy(scriptSupplier.apply(sockToStringForPython(acceptor.getAddress())));
		acceptor.setTimeout(CONNECT_TIMEOUT_MS);
		try {
			TraceRmiConnection connection = acceptor.accept();
			return new PyAndConnection(exec, connection);
		}
		catch (SocketTimeoutException e) {
			Msg.error(this, "Timed out waiting for client connection");
			exec.session.destroyForcibly();
			exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).handle();
			throw e;
		}
	}

	protected PyAndConnection startAndConnectPy() throws Exception {
		return startAndConnectPy(addr -> """
				%s
				c = connect(%s)
				""".formatted(PREAMBLE, addr));
	}

	@SuppressWarnings("resource")
	protected String runThrowError(Function<String, String> scriptSupplier)
			throws Exception {
		PyAndConnection conn = startAndConnectPy(scriptSupplier);
		PyResult r = conn.exec.future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
		String stdout = r.handle();
		waitForPass(() -> assertTrue(conn.connection.isClosed()));
		return stdout;
	}

	protected ManagedDomainObject<Trace> openTrace(String path) throws Exception {
		DomainFile df = env.getProject().getProjectData().getFile(path);
		assertNotNull(df);
		return new ManagedDomainObject<>(df, Trace.class, monitor);
	}

	protected ManagedDomainObject<Trace> waitTrace(String path) throws Exception {
		DomainFile df;
		long start = System.currentTimeMillis();
		while (true) {
			df = env.getProject().getProjectData().getFile(path);
			if (df != null) {
				return new ManagedDomainObject<>(df, Trace.class, monitor);
			}
			Thread.sleep(1000);
			if (System.currentTimeMillis() - start > 30000) {
				throw new TimeoutException("30 seconds expired waiting for domain file");
			}
		}
	}

	protected void waitTxDone() {
		waitFor(() -> tb.trace.getCurrentTransactionInfo() == null);
	}

	@Test
	public void testTomlVersionConsistency() throws IOException {
		List<String> toml = readToml("Debugger-rmi-trace");
		assertVersionMatchesApplication(parseVersionFromToml(toml));
	}

	@Test
	public void testConnect() throws Exception {
		runThrowError(addr -> """
				%s
				c = connect(%s)
				exit()
				""".formatted(PREAMBLE, addr));
	}

	@Test
	public void testClose() throws Exception {
		runThrowError(addr -> """
				%s
				c = connect(%s)
				c.close()
				exit()
				""".formatted(PREAMBLE, addr));
	}

	@Test
	public void testCreateTrace() throws Exception {
		runThrowError(addr -> """
				%s
				c = connect(%s)
				trace = c.create_trace("/test", "DATA:LE:64:default", "pointer64", extra=None)
				print(trace)
				exit()
				""".formatted(PREAMBLE, addr));
		try (ManagedDomainObject<Trace> obj = openTrace("/New Traces/test")) {
			// it worked and has the correct type
		}
	}

	@Test
	public void testMethodRegistrationAndInvocation() throws Exception {
		try (PyAndConnection pac = startAndConnectPy(addr -> """
				%s

				@registry.method()
				def py_eval(expr: str) -> str:
				    return repr(eval(expr))

				c = connect(%s)
				""".formatted(PREAMBLE, addr))) {
			RemoteMethod pyEval = pac.getMethod("py_eval");

			assertEquals(String.class,
				MinimalSchemaContext.INSTANCE.getSchema(pyEval.retType()).getType());
			assertEquals("expr", Unique.assertOne(pyEval.parameters().keySet()));
			assertEquals(String.class,
				MinimalSchemaContext.INSTANCE.getSchema(pyEval.parameters().get("expr").type())
						.getType());

			String result = (String) pyEval.invoke(Map.ofEntries(
				Map.entry("expr", "c")));
			assertThat(result, Matchers.startsWith("<ghidratrace.Client <socket.socket"));
		}
	}

	@Test
	public void testRegisterAnnotated() throws Exception {
		try (PyAndConnection pac = startAndConnectPy(addr -> """
				%s

				@registry.method()
				def py_eval(expr: Annotated[str, ParamDesc(display="Expression")]) -> Annotated[
				    Any, ParamDesc(schema=sch.STRING)]:
				    return repr(eval(expr))

				c = connect(%s)
				""".formatted(PREAMBLE, addr))) {
			RemoteMethod pyEval = pac.getMethod("py_eval");

			assertEquals(String.class,
				MinimalSchemaContext.INSTANCE.getSchema(pyEval.retType()).getType());
			assertEquals("expr", Unique.assertOne(pyEval.parameters().keySet()));

			RemoteParameter param = pyEval.parameters().get("expr");
			assertEquals(String.class,
				MinimalSchemaContext.INSTANCE.getSchema(param.type()).getType());
			assertEquals("Expression", param.display());

			String result = (String) pyEval.invoke(Map.ofEntries(
				Map.entry("expr", "c")));
			assertThat(result, Matchers.startsWith("<ghidratrace.Client <socket.socket"));
		}
	}

	@Test
	public void testRegisterOptional() throws Exception {
		try (PyAndConnection pac = startAndConnectPy(addr -> """
				%s

				@registry.method()
				def py_eval(expr: Optional[str]) -> Optional[str]:
				    return repr(eval(expr))

				c = connect(%s)
				""".formatted(PREAMBLE, addr))) {
			RemoteMethod pyEval = pac.getMethod("py_eval");

			assertEquals(String.class,
				MinimalSchemaContext.INSTANCE.getSchema(pyEval.retType()).getType());
			assertEquals("expr", Unique.assertOne(pyEval.parameters().keySet()));

			RemoteParameter param = pyEval.parameters().get("expr");
			assertEquals(String.class,
				MinimalSchemaContext.INSTANCE.getSchema(param.type()).getType());

			String result = (String) pyEval.invoke(Map.ofEntries(
				Map.entry("expr", "c")));
			assertThat(result, Matchers.startsWith("<ghidratrace.Client <socket.socket"));
		}
	}

	@Test
	public void testRegisterObject() throws Exception {
		try (PyAndConnection pac = startAndConnectPy(addr -> """
				%s

				class Session(TraceObject):
				    pass

				@registry.method()
				def py_eval(session: Session, expr: str) -> str:
				    return repr(eval(expr))

				c = connect(%s)
				""".formatted(PREAMBLE, addr))) {
			RemoteMethod pyEval = pac.getMethod("py_eval");

			assertEquals(String.class,
				MinimalSchemaContext.INSTANCE.getSchema(pyEval.retType()).getType());
			assertEquals(Set.of("session", "expr"), pyEval.parameters().keySet());

			RemoteParameter param = pyEval.parameters().get("session");
			assertEquals(new SchemaName("Session"), param.type());
		}
	}

	@Test
	public void testRegisterObjectBad() throws Exception {
		String out = runThrowError(addr -> """
				%s
				c = connect(%s)

				class Session(object):
				    pass

				def py_eval(session: Session, expr: str) -> str:
				    return repr(eval(expr))

				try:
				    registry.method()(py_eval)
				except TypeError as e:
				    print(f"---ERR:{e}---")

				exit()
				""".formatted(PREAMBLE, addr));
		assertThat(out, Matchers.containsString(
			"---ERR:Cannot get schema for <class '__main__.Session'>---"));
	}

	@Test
	public void testSnapshotDefaultNoTx() throws Exception {
		String out = runThrowError(addr -> """
				%s
				c = connect(%s)
				trace = c.create_trace("/test", "DATA:LE:64:default", "pointer64", extra=None)

				try:
				    trace.snapshot("Test")
				    raise Exception("Expected error")
				except TraceRmiError as e:
				    print(f"---ERR:{e}---")

				exit()
				""".formatted(PREAMBLE, addr));
		assertThat(out,
			Matchers.containsString("---ERR:%s".formatted(NoTransactionException.class.getName())));
	}

	@Test
	public void testSnapshotDefault() throws Exception {
		runThrowError(addr -> """
				%s
				c = connect(%s)
				trace = c.create_trace("/test", "DATA:LE:64:default", "pointer64", extra=None)

				with trace.open_tx("Create snapshot") as tx:
				    trace.snapshot("Test")

				exit()
				""".formatted(PREAMBLE, addr));
		try (ManagedDomainObject<Trace> obj = openTrace("/New Traces/test")) {
			Trace trace = obj.get();
			TraceSnapshot snapshot = trace.getTimeManager().getSnapshot(0, false);
			assertEquals("Test", snapshot.getDescription());
		}
	}

	@Test
	public void testSnapshotSnapOnly() throws Exception {
		runThrowError(addr -> """
				%s
				c = connect(%s)
				trace = c.create_trace("/test", "DATA:LE:64:default", "pointer64", extra=None)

				with trace.open_tx("Create snapshot") as tx:
				    trace.snapshot("Test", time=Schedule(10, 0))

				exit()
				""".formatted(PREAMBLE, addr));
		try (ManagedDomainObject<Trace> obj = openTrace("/New Traces/test")) {
			Trace trace = obj.get();
			TraceSnapshot snapshot = trace.getTimeManager().getSnapshot(10, false);
			assertEquals("Test", snapshot.getDescription());
		}
	}

	protected Matcher matchOne(String out, Pattern pat) {
		return Unique.assertOne(out.lines().map(pat::matcher).filter(Matcher::find));
	}

	@Test
	public void testSnapshotSchedule() throws Exception {
		String out = runThrowError(addr -> """
				%s
				c = connect(%s)
				trace = c.create_trace("/test", "DATA:LE:64:default", "pointer64", extra=None)

				with trace.open_tx("Create snapshot") as tx:
				    snap = trace.snapshot("Test", time=Schedule(10, 500))
				    print(f"---SNAP:{snap}---")

				exit()
				""".formatted(PREAMBLE, addr));

		long snap = Long.parseLong(matchOne(out, Pattern.compile("---SNAP:(-?\\d*)---")).group(1));
		assertThat(snap, Matchers.lessThan(0L));
		try (ManagedDomainObject<Trace> obj = openTrace("/New Traces/test")) {
			Trace trace = obj.get();
			TraceSnapshot snapshot = trace.getTimeManager().getSnapshot(snap, false);
			assertEquals("Test", snapshot.getDescription());
		}
	}

	@Test
	public void testSnapshotScheduleInBatch() throws Exception {
		String out = runThrowError(addr -> """
				%s
				c = connect(%s)
				trace = c.create_trace("/test", "DATA:LE:64:default", "pointer64", extra=None)

				with trace.open_tx("Create snapshot") as tx:
				    with c.batch() as b:
				        snap = trace.snapshot("Test", time=Schedule(10, 500))
				        print(f"---SNAP:{snap}---")

				exit()
				""".formatted(PREAMBLE, addr));

		long snap = Long.parseLong(matchOne(out, Pattern.compile("---SNAP:(-?\\d*)---")).group(1));
		assertThat(snap, Matchers.lessThan(0L));
		try (ManagedDomainObject<Trace> obj = openTrace("/New Traces/test")) {
			Trace trace = obj.get();
			TraceSnapshot snapshot = trace.getTimeManager().getSnapshot(snap, false);
			assertEquals("Test", snapshot.getDescription());
		}
	}
}