Skip to content

VERY experimental Nintendo Switch 2 emulator

C++

use faster submodule urls (github etc) (#143)

swurl2a8ff1a59c5d
diff --git a/.gitmodules b/.gitmodules
index 43ecc99ebd..b508e2247d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -3,43 +3,43 @@
[submodule "cubeb"]
path = externals/cubeb
- url = https://git.eden-emu.dev/eden-emu/cubeb.git
+ url = https://github.com/mozilla/cubeb.git
[submodule "libusb"]
path = externals/libusb/libusb
- url = https://git.eden-emu.dev/eden-emu/libusb.git
+ url = https://github.com/libusb/libusb.git
[submodule "Vulkan-Headers"]
path = externals/Vulkan-Headers
- url = https://git.eden-emu.dev/eden-emu/Vulkan-Headers.git
+ url = https://github.com/KhronosGroup/Vulkan-Headers.git
[submodule "xbyak"]
path = externals/xbyak
- url = https://git.eden-emu.dev/eden-emu/xbyak.git
+ url = https://github.com/herumi/xbyak.git
[submodule "opus"]
path = externals/opus
- url = https://git.eden-emu.dev/eden-emu/opus.git
+ url = https://github.com/xiph/opus.git
[submodule "SDL"]
path = externals/SDL
- url = https://git.eden-emu.dev/eden-emu/SDL.git
+ url = https://github.com/libsdl-org/SDL.git
[submodule "cpp-httplib"]
path = externals/cpp-httplib
- url = https://git.eden-emu.dev/eden-emu/cpp-httplib.git
+ url = https://github.com/yhirose/cpp-httplib.git
[submodule "ffmpeg"]
path = externals/ffmpeg/ffmpeg
- url = https://git.eden-emu.dev/eden-emu/FFmpeg.git
+ url = https://git.ffmpeg.org/ffmpeg.git
[submodule "vcpkg"]
path = externals/vcpkg
- url = https://git.eden-emu.dev/eden-emu/vcpkg.git
+ url = https://github.com/microsoft/vcpkg.git
[submodule "cpp-jwt"]
path = externals/cpp-jwt
- url = https://git.eden-emu.dev/eden-emu/cpp-jwt.git
+ url = https://github.com/arun11299/cpp-jwt.git
[submodule "VulkanMemoryAllocator"]
path = externals/VulkanMemoryAllocator
- url = https://git.eden-emu.dev/eden-emu/VulkanMemoryAllocator.git
+ url = https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator.git
[submodule "Vulkan-Utility-Libraries"]
path = externals/Vulkan-Utility-Libraries
- url = https://git.eden-emu.dev/eden-emu/Vulkan-Utility-Libraries.git
+ url = https://github.com/KhronosGroup/Vulkan-Utility-Libraries.git
[submodule "externals/boost-headers"]
path = externals/boost-headers
- url = https://git.eden-emu.dev/eden-emu/headers.git
+ url = https://github.com/boostorg/headers.git
[submodule "externals/dynarmic/externals/catch"]
path = externals/dynarmic/externals/catch
url = https://github.com/catchorg/Catch2.git
diff --git a/externals/libadrenotools/lib/linkernsbypass/CMakeLists.txt b/externals/libadrenotools/lib/linkernsbypass/CMakeLists.txt
new file mode 100644
index 0000000000..c2e92b0422
--- /dev/null
+++ b/externals/libadrenotools/lib/linkernsbypass/CMakeLists.txt
@@ -0,0 +1,19 @@
+cmake_minimum_required(VERSION 3.14)
+
+if(NOT ${CMAKE_ANDROID_ARCH_ABI} STREQUAL arm64-v8a)
+ message(FATAL_ERROR "Unsupported target architecture: ${CMAKE_ANDROID_ARCH_ABI}. Please make an issue on the repo!")
+endif()
+
+project(linkernsbypass LANGUAGES CXX)
+
+set(SOURCES android_linker_ns.cpp
+ android_linker_ns.h
+ elf_soname_patcher.cpp
+ elf_soname_patcher.h)
+
+add_library(linkernsbypass STATIC ${SOURCES})
+
+
+target_compile_options(linkernsbypass PRIVATE -Wall -Wextra)
+target_link_libraries(linkernsbypass android dl)
+target_include_directories(linkernsbypass PUBLIC .)
diff --git a/externals/libadrenotools/lib/linkernsbypass/LICENSE b/externals/libadrenotools/lib/linkernsbypass/LICENSE
new file mode 100644
index 0000000000..a8f664a745
--- /dev/null
+++ b/externals/libadrenotools/lib/linkernsbypass/LICENSE
@@ -0,0 +1,24 @@
+BSD 2-Clause License
+
+Copyright (c) 2021, Billy Laws
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/externals/libadrenotools/lib/linkernsbypass/README.md b/externals/libadrenotools/lib/linkernsbypass/README.md
new file mode 100644
index 0000000000..264f23a39a
--- /dev/null
+++ b/externals/libadrenotools/lib/linkernsbypass/README.md
@@ -0,0 +1,9 @@
+### Android Linker Namespace Bypass Library
+Provides access to the hidden linker namespace functionality in Android 9+ and exposes an interface for hooking libraries similar to a runtime equivalent of `LD_PRELOAD`.
+See the `android_linker_ns.h` header for an API reference.
+
+#### Support
+Android 9+
+Arm64
+
+Android 8 and arm32 could be supported with some trivial changes, feel free to open an issue if you have a use for this library on either of them.
diff --git a/externals/libadrenotools/lib/linkernsbypass/android_linker_ns.cpp b/externals/libadrenotools/lib/linkernsbypass/android_linker_ns.cpp
new file mode 100644
index 0000000000..3bd58992c2
--- /dev/null
+++ b/externals/libadrenotools/lib/linkernsbypass/android_linker_ns.cpp
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: BSD-2-Clause
+// Copyright © 2021 Billy Laws
+
+#include <array>
+#include <cstdio>
+#include <cstdlib>
+#include <errno.h>
+#include <dlfcn.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <android/dlext.h>
+#include <android/log.h>
+#include <android/api-level.h>
+#include <sys/mman.h>
+#include "elf_soname_patcher.h"
+#include "android_linker_ns.h"
+
+using loader_android_create_namespace_t = android_namespace_t *(*)(const char *, const char *, const char *, uint64_t, const char *, android_namespace_t *, const void *);
+static loader_android_create_namespace_t loader_android_create_namespace;
+
+static bool lib_loaded;
+
+
+/* Public API */
+bool linkernsbypass_load_status() {
+ return lib_loaded;
+}
+
+struct android_namespace_t *android_create_namespace(const char *name,
+ const char *ld_library_path,
+ const char *default_library_path,
+ uint64_t type,
+ const char *permitted_when_isolated_path,
+ android_namespace_t *parent_namespace) {
+ auto caller{__builtin_return_address(0)};
+ return loader_android_create_namespace(name, ld_library_path, default_library_path, type,
+ permitted_when_isolated_path, parent_namespace, caller);
+}
+
+struct android_namespace_t *android_create_namespace_escape(const char *name,
+ const char *ld_library_path,
+ const char *default_library_path,
+ uint64_t type,
+ const char *permitted_when_isolated_path,
+ android_namespace_t *parent_namespace) {
+ auto caller{reinterpret_cast<void *>(&dlopen)};
+ return loader_android_create_namespace(name, ld_library_path, default_library_path, type,
+ permitted_when_isolated_path, parent_namespace, caller);
+}
+
+android_get_exported_namespace_t android_get_exported_namespace;
+
+android_link_namespaces_all_libs_t android_link_namespaces_all_libs;
+
+android_link_namespaces_t android_link_namespaces;
+
+bool linkernsbypass_link_namespace_to_default_all_libs(android_namespace_t *to) {
+ // Creating a shared namespace with the default parent will give a copy of the default namespace that we can actually access
+ // This is needed since there is no way to access a direct handle to the default namespace as it's not exported
+ static auto defaultNs{android_create_namespace_escape("default_copy", nullptr, nullptr, ANDROID_NAMESPACE_TYPE_SHARED, nullptr, nullptr)};
+ return android_link_namespaces_all_libs(to, defaultNs);
+}
+
+void *linkernsbypass_namespace_dlopen(const char *filename, int flags, android_namespace_t *ns) {
+ android_dlextinfo extInfo{
+ .flags = ANDROID_DLEXT_USE_NAMESPACE,
+ .library_namespace = ns
+ };
+
+ return android_dlopen_ext(filename, flags, &extInfo);
+}
+
+#ifndef __NR_memfd_create
+ #if defined(__aarch64__)
+ #define __NR_memfd_create 279
+ #else
+ #error Unsupported target architecture!
+ #endif
+#endif
+
+void *linkernsbypass_namespace_dlopen_unique(const char *libPath, const char *libTargetDir, int flags, android_namespace_t *ns) {
+ static std::array<char, PATH_MAX> PathBuf{};
+
+ // Used as a unique ID for overwriting soname and creating target lib files
+ static uint16_t TargetId{};
+
+ int libTargetFd{[&] () {
+ if (libTargetDir) {
+ snprintf(PathBuf.data(), PathBuf.size(), "%s/%d_patched.so", libTargetDir, TargetId);
+ return open(PathBuf.data(), O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
+ } else {
+ // If memfd isn't supported errno will contain ENOSYS after calling
+ errno = 0;
+ int fd{static_cast<int>(syscall(__NR_memfd_create, libPath, 0))};
+ if (errno == ENOSYS || fd < 0)
+ return -1;
+ else
+ return fd;
+ }
+ }()};
+ if (libTargetFd == -1)
+ return nullptr;
+
+ // Partially overwrite soname with 3 digits (replacing lib...) with to make sure a cached so isn't loaded
+ std::array<char, 3> sonameOverwrite{};
+ snprintf(sonameOverwrite.data(), sonameOverwrite.size(), "%03u", TargetId++);
+
+ if (!elf_soname_patch(libPath, libTargetFd, sonameOverwrite.data()))
+ return nullptr;
+
+ // Load our patched library into the hook namespace
+ android_dlextinfo hookExtInfo{
+ .flags = ANDROID_DLEXT_USE_NAMESPACE | ANDROID_DLEXT_USE_LIBRARY_FD,
+ .library_fd = libTargetFd,
+ .library_namespace = ns
+ };
+
+ // Make a path that looks about right
+ snprintf(PathBuf.data(), PathBuf.size(), "/proc/self/fd/%d", libTargetFd);
+
+ return android_dlopen_ext(PathBuf.data(), flags, &hookExtInfo);
+}
+
+static void *align_ptr(void *ptr) {
+ return reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(ptr) & ~(getpagesize() - 1));
+}
+
+/* Private */
+__attribute__((constructor)) static void resolve_linker_symbols() {
+ using loader_dlopen_t = void *(*)(const char *, int, const void *);
+
+ if (android_get_device_api_level() < 28)
+ return;
+
+ // ARM64 specific function walking to locate the internal dlopen handler
+ auto loader_dlopen{[]() {
+ union BranchLinked {
+ uint32_t raw;
+
+ struct {
+ int32_t offset : 26; //!< 26-bit branch offset
+ uint8_t sig : 6; //!< 6-bit signature
+ };
+
+ bool Verify() {
+ return sig == 0x25;
+ }
+ };
+ static_assert(sizeof(BranchLinked) == 4, "BranchLinked is wrong size");
+
+ // Some devices ship with --X mapping for exexecutables so work around that
+ mprotect(align_ptr(reinterpret_cast<void *>(&dlopen)), getpagesize(), PROT_WRITE | PROT_READ | PROT_EXEC);
+
+ // dlopen is just a wrapper for __loader_dlopen that passes the return address as the third arg hence we can just walk it to find __loader_dlopen
+ auto blInstr{reinterpret_cast<BranchLinked *>(&dlopen)};
+ while (!blInstr->Verify())
+ blInstr++;
+
+ return reinterpret_cast<loader_dlopen_t>(blInstr + blInstr->offset);
+ }()};
+
+ // Protect the loader_dlopen function to remove the BTI attribute (since this is an internal function that isn't intended to be jumped indirectly to)
+ mprotect(align_ptr(reinterpret_cast<void *>(&loader_dlopen)), getpagesize(), PROT_WRITE | PROT_READ | PROT_EXEC);
+
+ // Passing dlopen as a caller address tricks the linker into using the internal unrestricted namespace letting us access libraries that are normally forbidden in the classloader namespace imposed on apps
+ auto ldHandle{loader_dlopen("ld-android.so", RTLD_LAZY, reinterpret_cast<void *>(&dlopen))};
+ if (!ldHandle)
+ return;
+
+ android_link_namespaces_all_libs = reinterpret_cast<android_link_namespaces_all_libs_t>(dlsym(ldHandle, "__loader_android_link_namespaces_all_libs"));
+ if (!android_link_namespaces_all_libs)
+ return;
+
+ android_link_namespaces = reinterpret_cast<android_link_namespaces_t>(dlsym(ldHandle, "__loader_android_link_namespaces"));
+ if (!android_link_namespaces)
+ return;
+
+ auto libdlAndroidHandle{loader_dlopen("libdl_android.so", RTLD_LAZY, reinterpret_cast<void *>(&dlopen))};
+ if (!libdlAndroidHandle)
+ return;
+
+ loader_android_create_namespace = reinterpret_cast<loader_android_create_namespace_t>(dlsym(libdlAndroidHandle, "__loader_android_create_namespace"));
+ if (!loader_android_create_namespace)
+ return;
+
+ android_get_exported_namespace = reinterpret_cast<android_get_exported_namespace_t>(dlsym(libdlAndroidHandle, "__loader_android_get_exported_namespace"));
+ if (!android_get_exported_namespace)
+ return;
+
+ // Lib is now safe to use
+ lib_loaded = true;
+}
diff --git a/externals/libadrenotools/lib/linkernsbypass/android_linker_ns.h b/externals/libadrenotools/lib/linkernsbypass/android_linker_ns.h
new file mode 100644
index 0000000000..77ec4005a5
--- /dev/null
+++ b/externals/libadrenotools/lib/linkernsbypass/android_linker_ns.h
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: BSD-2-Clause
+// Copyright © 2021 Billy Laws
+
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+// https://cs.android.com/android/platform/superproject/+/0a492a4685377d41fef2b12e9af4ebfa6feef9c2:art/libnativeloader/include/nativeloader/dlext_namespaces.h;l=25;bpv=1;bpt=1
+enum {
+ ANDROID_NAMESPACE_TYPE_REGULAR = 0,
+ ANDROID_NAMESPACE_TYPE_ISOLATED = 1,
+ ANDROID_NAMESPACE_TYPE_SHARED = 2,
+ ANDROID_NAMESPACE_TYPE_EXEMPT_LIST_ENABLED = 0x08000000,
+ ANDROID_NAMESPACE_TYPE_ALSO_USED_AS_ANONYMOUS = 0x10000000,
+ ANDROID_NAMESPACE_TYPE_SHARED_ISOLATED = ANDROID_NAMESPACE_TYPE_SHARED | ANDROID_NAMESPACE_TYPE_ISOLATED,
+};
+
+/**
+ * @brief Checks if linkernsbypass loaded successfully and is safe to use
+ * @note IMPORTANT: This should be called before any calls to the rest of the library are made
+ * @return true if loading succeeded
+ */
+bool linkernsbypass_load_status();
+
+// https://cs.android.com/android/platform/superproject/+/0a492a4685377d41fef2b12e9af4ebfa6feef9c2:art/libnativeloader/include/nativeloader/dlext_namespaces.h;l=86;bpv=1;bpt=1
+struct android_namespace_t *android_create_namespace(const char *name,
+ const char *ld_library_path,
+ const char *default_library_path,
+ uint64_t type,
+ const char *permitted_when_isolated_path,
+ struct android_namespace_t *parent_namespace);
+
+struct android_namespace_t *android_create_namespace_escape(const char *name,
+ const char *ld_library_path,
+ const char *default_library_path,
+ uint64_t type,
+ const char *permitted_when_isolated_path,
+ struct android_namespace_t *parent_namespace);
+
+// https://cs.android.com/android/platform/superproject/+/dcb01ef31026b3b8aeb72dada3370af63fe66bbd:bionic/linker/linker.cpp;l=3554
+typedef struct android_namespace_t *(*android_get_exported_namespace_t)(const char *);
+extern android_get_exported_namespace_t android_get_exported_namespace;
+
+// https://cs.android.com/android/platform/superproject/+/dcb01ef31026b3b8aeb72dada3370af63fe66bbd:bionic/linker/linker.cpp;l=2499
+typedef bool (*android_link_namespaces_all_libs_t)(struct android_namespace_t *, struct android_namespace_t *);
+extern android_link_namespaces_all_libs_t android_link_namespaces_all_libs;
+
+// https://cs.android.com/android/platform/superproject/+/dcb01ef31026b3b8aeb72dada3370af63fe66bbd:bionic/linker/linker.cpp;l=2473
+typedef bool (*android_link_namespaces_t)(struct android_namespace_t *, struct android_namespace_t *, const char *);
+extern android_link_namespaces_t android_link_namespaces;
+
+/**
+ * @brief Like android_link_namespaces_all_libs but links from the default namespace
+ */
+bool linkernsbypass_link_namespace_to_default_all_libs(struct android_namespace_t *to);
+
+/**
+ * @brief Loads a library into a namespace
+ * @note IMPORTANT: If `filename` is compiled with the '-z global' linker flag and RTLD_GLOBAL is supplied in `flags` the library will be added to the namespace's LD_PRELOAD list
+ * @param filename The name of the library to load
+ * @param flags The rtld flags for `filename`
+ * @param ns The namespace to dlopen into
+ */
+void *linkernsbypass_namespace_dlopen(const char *filename, int flags, struct android_namespace_t *ns);
+
+/**
+ * @brief Force loads a unique instance of a library into a namespace
+ * @param libPath The path to the library to load with hooks applied
+ * @param libTargetDir A temporary directory to hold the soname patched library at `libPath`, will attempt to use memfd if nullptr
+ * @param flags The rtld flags for `libName`
+ * @param ns The namespace to dlopen into
+ */
+void *linkernsbypass_namespace_dlopen_unique(const char *libPath, const char *libTargetDir, int flags, struct android_namespace_t *ns);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/externals/libadrenotools/lib/linkernsbypass/elf_soname_patcher.cpp b/externals/libadrenotools/lib/linkernsbypass/elf_soname_patcher.cpp
new file mode 100644
index 0000000000..db619ab534
--- /dev/null
+++ b/externals/libadrenotools/lib/linkernsbypass/elf_soname_patcher.cpp
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: BSD-2-Clause
+// Copyright © 2021 Billy Laws
+
+#include <initializer_list>
+#include <cstdint>
+#include <errno.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <link.h>
+#include <elf.h>
+#include "elf_soname_patcher.h"
+
+bool elf_soname_patch(const char *libPath, int targetFd, const char *sonamePatch) {
+ struct stat libStat{};
+ if (stat(libPath, &libStat))
+ return false;
+
+ if (ftruncate(targetFd, libStat.st_size) == -1)
+ return false;
+
+ // Map the memory so we can read our elf into it
+ auto mappedLib{reinterpret_cast<uint8_t *>(mmap(nullptr, libStat.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, targetFd, 0))};
+ if (!mappedLib)
+ return false;
+
+ int libFd{open(libPath, O_RDONLY)};
+ if (libFd == false)
+ return false;
+
+ // Read lib elf into target file
+ if (read(libFd, mappedLib, libStat.st_size) != libStat.st_size)
+ return false;
+
+ // No longer needed
+ close(libFd);
+
+ auto eHdr{reinterpret_cast<ElfW(Ehdr) *>(mappedLib)};
+ auto sHdrEntries{reinterpret_cast<ElfW(Shdr) *>(mappedLib + eHdr->e_shoff)};
+
+ // Iterate over section headers to find the .dynamic section
+ for (ElfW(Half) i{}; i < eHdr->e_shnum; i++) {
+ auto &sHdr{sHdrEntries[i]};
+ if (sHdr.sh_type == SHT_DYNAMIC) {
+ auto strTab{reinterpret_cast<char *>(mappedLib + sHdrEntries[sHdr.sh_link].sh_offset)};
+ auto dynHdrEntries{reinterpret_cast<ElfW(Dyn) *>(mappedLib + sHdr.sh_offset)};
+
+ // Iterate over .dynamic entries to find DT_SONAME
+ for (ElfW(Xword) k{}; k < (sHdr.sh_size / sHdr.sh_entsize); k++) {
+ auto &dynHdrEntry{dynHdrEntries[k]};
+ if (dynHdrEntry.d_tag == DT_SONAME) {
+ char *soname{strTab + dynHdrEntry.d_un.d_val};
+
+ // Partially replace the old soname with the soname patch
+ size_t charIdx{};
+ for (; soname[charIdx] != 0 && sonamePatch[charIdx] != 0; charIdx++)
+ soname[charIdx] = sonamePatch[charIdx];
+
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+}
diff --git a/externals/libadrenotools/lib/linkernsbypass/elf_soname_patcher.h b/externals/libadrenotools/lib/linkernsbypass/elf_soname_patcher.h
new file mode 100644
index 0000000000..0b2c186ffc
--- /dev/null
+++ b/externals/libadrenotools/lib/linkernsbypass/elf_soname_patcher.h
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: BSD-2-Clause
+// Copyright © 2021 Billy Laws
+
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Overwrites a portion of the soname in an elf by loading it into shared memory and modifying .dynstr
+ * @note IMPORTANT: The supplied soname patch will overwrite the first strlen(sonamePatch) chars of the soname
+ * @param elfPath Full path to the elf to patch
+ * @param targetFd FD to use for storing the patched library
+ * @return True on success
+ */
+bool elf_soname_patch(const char *elfPath, int targetFd, const char *newSoname);
+
+#ifdef __cplusplus
+}
+#endif