Skip to content

armgit / xtool

Public Imported from GitHub
0

Cross-platform Xcode replacement. Build and deploy iOS apps with SwiftPM on Linux, Windows, macOS.

SDK builder initial pass

Kabir Oberaibaf71fa1da6e
diff --git a/Dockerfile b/Dockerfile
index 046b74a..b3f51e2 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
# Note: We use 20.04 since AppImage recommends building on the
# oldest configuration that you support
-FROM swift:6.0-focal AS build-base
+FROM swift:6.1-focal AS build-base
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
@@ -17,6 +17,7 @@ RUN apt-get update \
libxml2 \
curl libcurl4-openssl-dev \
zip unzip \
+ liblzma-dev zlib1g-dev \
&& rm -rf /var/lib/apt/lists/*
diff --git a/Package.resolved b/Package.resolved
index 8654e97..a70ee8d 100644
--- a/Package.resolved
+++ b/Package.resolved
@@ -1,5 +1,5 @@
{
- "originHash" : "360bc3ae9f3e8613a4fb4b6583a90d4819977ed68df921d928f49717dad97299",
+ "originHash" : "7d8342a181303ec948ba0a80e3bdadfdefbde5ca30e750282a9caf834410fae7",
"pins" : [
{
"identity" : "aexml",
@@ -334,6 +334,15 @@
"version" : "1.3.1"
}
},
+ {
+ "identity" : "unxip",
+ "kind" : "remoteSourceControl",
+ "location" : "https://github.com/saagarjha/unxip",
+ "state" : {
+ "revision" : "d6f0009109f7502f8f5b3f23dcd5b43cf5b29b32",
+ "version" : "3.1.0"
+ }
+ },
{
"identity" : "version",
"kind" : "remoteSourceControl",
diff --git a/Package.swift b/Package.swift
index 1e3c459..3fe0c05 100644
--- a/Package.swift
+++ b/Package.swift
@@ -51,6 +51,7 @@ let package = Package(
.package(url: "https://github.com/attaswift/BigInt", from: "5.5.0"),
.package(url: "https://github.com/mxcl/Version", from: "2.1.0"),
.package(url: "https://github.com/jpsim/Yams", from: "5.1.3"),
+ .package(url: "https://github.com/saagarjha/unxip", from: "3.1.0"),
// TODO: just depend on tuist/XcodeProj instead
.package(url: "https://github.com/yonaskolb/XcodeGen", from: "2.43.0"),
@@ -109,6 +110,12 @@ let package = Package(
],
cSettings: cSettings
),
+ .testTarget(
+ name: "SupersignCLITests",
+ dependencies: [
+ "SupersignCLISupport",
+ ]
+ ),
.testTarget(
name: "SupersignTests",
dependencies: [
@@ -133,6 +140,7 @@ let package = Package(
.product(name: "SystemPackage", package: "swift-system"),
.product(name: "NIOPosix", package: "swift-nio"),
.product(name: "Version", package: "Version"),
+ .product(name: "libunxip", package: "unxip"),
],
cSettings: cSettings
),
diff --git a/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swift b/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swift
index 78a51d3..4e59a87 100644
--- a/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swift
+++ b/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swift
@@ -12,11 +12,69 @@ struct DevSDKCommand: AsyncParsableCommand {
subcommands: [
DevSDKInstallCommand.self,
DevSDKRemoveCommand.self,
+ DevSDKBuildCommand.self,
],
defaultSubcommand: DevSDKInstallCommand.self
)
}
+struct DevSDKBuildCommand: AsyncParsableCommand {
+ static let configuration = CommandConfiguration(
+ commandName: "build",
+ abstract: "Build the Darwin SDK from Xcode.xip"
+ )
+
+ @Argument(
+ help: "Path to Xcode.xip or Xcode.app",
+ completion: .file(extensions: ["xip", "app"])
+ )
+ var path: String
+
+ @Argument(
+ help: "Output directory"
+ )
+ var outputDir: String
+
+ @Option(
+ help: ArgumentHelp(
+ "The architecture of the Linux host the SDK is being built for.",
+ discussion: "Defaults to 'auto', which attempts to match the current host architecture."
+ )
+ ) var arch: ArchSelection = .auto
+
+ func run() async throws {
+ let builderArch: SDKBuilder.Arch = switch arch {
+ case .auto:
+ #if arch(arm64)
+ .aarch64
+ #elseif arch(x86_64)
+ .x86_64
+ #else
+ throw Console.Error("Could not auto-detect target architecture. Please specify one with '--arch'.")
+ #endif
+ case .arm64: .aarch64
+ case .x86_64: .x86_64
+ }
+
+ let input: SDKBuilder.Input = if path.hasSuffix(".xip") {
+ .xip(path)
+ } else if path.hasSuffix(".app") || path.hasSuffix(".app/") {
+ .app(path)
+ } else {
+ throw Console.Error("Expected input path to end in .xip or .app")
+ }
+
+ let builder = SDKBuilder(input: input, outputPath: outputDir, arch: builderArch)
+ try await builder.buildSDK()
+ }
+}
+
+enum ArchSelection: String, ExpressibleByArgument {
+ case auto
+ case x86_64
+ case arm64
+}
+
struct DevSDKInstallCommand: AsyncParsableCommand {
static let configuration = CommandConfiguration(
commandName: "install",
diff --git a/Sources/SupersignCLISupport/DevCommands/SDKBuilder.swift b/Sources/SupersignCLISupport/DevCommands/SDKBuilder.swift
new file mode 100644
index 0000000..3ad57ef
--- /dev/null
+++ b/Sources/SupersignCLISupport/DevCommands/SDKBuilder.swift
@@ -0,0 +1,441 @@
+import Foundation
+import Dependencies
+import SystemPackage
+import libunxip
+import Supersign // HTTPClient, stdoutSafe
+import PackLib // ToolRegistry
+
+struct SDKBuilder {
+ enum Arch: String {
+ case x86_64
+ case aarch64
+ }
+
+ enum Input {
+ case xip(String)
+ case app(String)
+ }
+
+ let input: Input
+ let outputPath: String
+ let arch: Arch
+
+ func buildSDK() async throws {
+ // tag from https://github.com/kabiroberai/darwin-tools-linux-llvm
+ let darwinToolsVersion = "1.0.1"
+
+ // TODO: store relevant info for staleness check
+ let sdkVersion = "develop"
+
+ print("[Cleaning stage]")
+
+ let output = URL(fileURLWithPath: outputPath, isDirectory: true)
+ .appendingPathComponent("darwin.artifactbundle")
+
+ try? FileManager.default.removeItem(at: output)
+ try FileManager.default.createDirectory(
+ at: output,
+ withIntermediateDirectories: true
+ )
+
+ let toolsetDir = output.appendingPathComponent("toolset")
+
+ try FileManager.default.createDirectory(
+ at: toolsetDir,
+ withIntermediateDirectories: false
+ )
+
+ let pipe = Pipe()
+ let untar = Process()
+ untar.currentDirectoryURL = toolsetDir
+ untar.executableURL = try await ToolRegistry.locate("tar")
+ untar.arguments = ["xzf", "-"]
+ untar.standardInput = pipe.fileHandleForReading
+ async let tarExit: Void = untar.runUntilExit()
+
+ @Dependency(\.httpClient) var httpClient
+ let url = URL(string: """
+ https://github.com/kabiroberai/darwin-tools-linux-llvm/releases/download/\
+ v\(darwinToolsVersion)/toolset-\(arch.rawValue).tar.gz
+ """)!
+ let (response, body) = try await httpClient.send(HTTPRequest(url: url))
+ guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") }
+ let length: Int64? = switch body.length {
+ case .known(let known): known
+ case .unknown: nil
+ }
+ let writer = pipe.fileHandleForWriting
+ var written: Int64 = 0
+ for try await chunk in body {
+ try writer.write(contentsOf: chunk)
+ written += Int64(chunk.count)
+ if let length {
+ let progress = Int(Double(written) / Double(length) * 100)
+ print("\r[Installing toolchain] \(progress)%", terminator: "")
+ fflush(stdoutSafe)
+ }
+ }
+ print()
+ try writer.close()
+ try await tarExit
+
+ let dev = output.appendingPathComponent("Developer")
+
+ let appDir: URL
+ let cleanupStageDir: URL?
+ let wanted: Int?
+
+ switch input {
+ case .xip(let inputPath):
+ let devStage = output.appendingPathComponent("DeveloperStage")
+ try FileManager.default.createDirectory(at: devStage, withIntermediateDirectories: false)
+ wanted = try await extractXIP(inputPath: inputPath, outDir: devStage.path)
+ appDir = devStage.appendingPathComponent("Xcode.app")
+ cleanupStageDir = devStage
+ case .app(let appPath):
+ wanted = nil
+ appDir = URL(fileURLWithPath: appPath)
+ cleanupStageDir = nil
+ }
+
+ try FileManager.default.createDirectory(at: dev, withIntermediateDirectories: false)
+
+ var toDoDirs: [String] = ["Contents/Developer"]
+ var count = 0
+ while let next = toDoDirs.popLast() {
+ try Task.checkCancellation()
+
+ let url = appDir.appendingPathComponent(next)
+ let contents = try FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: [.isDirectoryKey])
+ for child in contents {
+ let path = "\(next)/\(child.lastPathComponent)"
+ guard Self.isWanted(path[...]) else { continue }
+
+ count += 1
+ if let wanted {
+ let progress = Int(Double(count) / Double(wanted) * 100)
+ print("\r[Installing SDKs] \(progress)%", terminator: "")
+ fflush(stdoutSafe)
+ }
+ if count % 100 == 0 {
+ if wanted == nil {
+ print("\r[Installing SDKs] Copied \(count) files", terminator: "")
+ fflush(stdoutSafe)
+ }
+ await Task.yield()
+ }
+
+ let insideDeveloper = path.dropFirst("Contents/Developer/".count)
+ guard !insideDeveloper.isEmpty else { continue }
+ let dest = dev.appendingPathComponent(String(insideDeveloper))
+
+ if try child.resourceValues(forKeys: [.isDirectoryKey]).isDirectory == true {
+ toDoDirs.append(path)
+ try FileManager.default.createDirectory(at: dest, withIntermediateDirectories: false)
+ } else {
+ try FileManager.default.copyItem(at: child, to: dest)
+ }
+ }
+ }
+
+ if wanted != nil {
+ // otherwise the last log might say 99%
+ print("\r[Installing SDKs] 100%", terminator: "")
+ }
+ print()
+
+ print("[Cleaning XIP]")
+ if let cleanupStageDir {
+ try? FileManager.default.removeItem(at: cleanupStageDir)
+ }
+
+ func sdk(platform: String, prefix: String) throws -> String {
+ let regex = try NSRegularExpression(pattern: #"^\#(prefix)\d+\.\d+\.sdk$"#)
+ let dir = dev.appendingPathComponent("Platforms/\(platform).platform/Developer/SDKs")
+ let names = try dir.contents().map(\.lastPathComponent)
+ guard let name = names.first(where: {
+ regex.firstMatch(in: $0, range: NSRange($0.startIndex..., in: $0)) != nil
+ }) else {
+ throw Console.Error("Could not find SDK for \(platform)/\(prefix)")
+ }
+ return name
+ }
+
+ let iPhoneOSSDK = try sdk(platform: "iPhoneOS", prefix: "iPhoneOS")
+ let iPhoneSimSDK = try sdk(platform: "iPhoneSimulator", prefix: "iPhoneSimulator")
+ let macOSSDK = try sdk(platform: "MacOSX", prefix: "MacOSX")
+
+ print("""
+ [Finalizing SDKs]
+ - \(iPhoneOSSDK)
+ - \(iPhoneSimSDK)
+ - \(macOSSDK)
+ """)
+
+ /*
+ XCTest and Testing.framework are located in *.platform/Developer/{Library/Frameworks,usr/lib} rather than inside
+ the SDK. These search paths are explicitly included when building tests, which presumably ensures that normal
+ applications don't accidentally link against them in production.
+
+ SwiftPM makes no such affordances outside of macOS, so we add the usr/lib path as include/library search paths
+ in the SDK config, and symlink the frameworks into the SDKs (since there's no frameworkSearchPaths option).
+ While this drops a safeguard it's better than not having the testing libs at all.
+ */
+
+ for platform in ["iPhoneOS", "MacOSX", "iPhoneSimulator"] {
+ let lib = "../../../../../Library"
+ let dest = dev.appendingPathComponent("""
+ Platforms/\(platform).platform/Developer/SDKs/\(platform).sdk\
+ /System/Library/Frameworks
+ """).path
+
+ try FileManager.default.createSymbolicLink(
+ atPath: "\(dest)/Testing.framework",
+ withDestinationPath: "\(lib)/Frameworks/Testing.framework"
+ )
+
+ try FileManager.default.createSymbolicLink(
+ atPath: "\(dest)/XCTest.framework",
+ withDestinationPath: "\(lib)/Frameworks/XCTest.framework"
+ )
+
+ try FileManager.default.createSymbolicLink(
+ atPath: "\(dest)/XCUIAutomation.framework",
+ withDestinationPath: "\(lib)/Frameworks/XCUIAutomation.framework"
+ )
+
+ try FileManager.default.createSymbolicLink(
+ atPath: "\(dest)/XCTestCore.framework",
+ withDestinationPath: "\(lib)/PrivateFrameworks/XCTestCore.framework"
+ )
+ }
+
+ print("[Writing metadata]")
+ try """
+ {
+ "schemaVersion": "1.0",
+ "artifacts": {
+ "darwin": {
+ "type": "swiftSDK",
+ "version": "0.0.1",
+ "variants": [
+ {
+ "path": ".",
+ "supportedTriples": ["aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]
+ }
+ ]
+ }
+ }
+ }
+ """.write(
+ to: output.appendingPathComponent("info.json"),
+ atomically: false,
+ encoding: .utf8
+ )
+
+ try """
+ {
+ "schemaVersion": "1.0",
+ "rootPath": "toolset/bin",
+ "linker": {
+ "path": "ld64.lld"
+ },
+ "swiftCompiler": {
+ "extraCLIOptions": [
+ "-use-ld=lld"
+ ]
+ }
+ }
+ """.write(
+ to: output.appendingPathComponent("toolset.json"),
+ atomically: false,
+ encoding: .utf8
+ )
+
+ func triple(platform: String, sdk: String) -> SDKDefinition.Triple {
+ SDKDefinition.Triple(
+ sdkRootPath: "Developer/Platforms/\(platform).platform/Developer/SDKs/\(sdk)",
+ includeSearchPaths: ["Developer/Platforms/\(platform).platform/Developer/usr/lib"],
+ librarySearchPaths: ["Developer/Platforms/\(platform).platform/Developer/usr/lib"],
+ swiftResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift",
+ swiftStaticResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static",
+ toolsetPaths: ["toolset.json"]
+ )
+ }
+
+ let sdkDefinition = SDKDefinition(
+ schemaVersion: "4.0",
+ targetTriples: [
+ "arm64-apple-ios": triple(platform: "iPhoneOS", sdk: iPhoneOSSDK),
+ "arm64-apple-ios-simulator": triple(platform: "iPhoneSimulator", sdk: iPhoneSimSDK),
+ "x86_64-apple-ios-simulator": triple(platform: "iPhoneSimulator", sdk: iPhoneSimSDK),
+ "arm64-apple-macosx": triple(platform: "MacOSX", sdk: macOSSDK),
+ "x86_64-apple-macosx": triple(platform: "MacOSX", sdk: macOSSDK),
+ ]
+ )
+
+ let encoder = JSONEncoder()
+ try encoder
+ .encode(sdkDefinition)
+ .write(to: output.appendingPathComponent("swift-sdk.json"))
+
+ try Data("\(sdkVersion)\n".utf8)
+ .write(to: output.appendingPathComponent("darwin-sdk-version.txt"))
+
+ print("Built SDK at \(output.path)")
+ }
+
+ // returns the number of files we actually want to keep,
+ // useful for computing progress % during fs traversal
+ private func extractXIP(inputPath: String, outDir: String) async throws -> Int {
+ let fd = try FileDescriptor.open(inputPath, .readOnly)
+ defer { try? fd.close() }
+
+ let length = try fd.seek(offset: 0, from: .end)
+ try fd.seek(offset: 0, from: .start)
+
+ // global state, ah well
+ let oldDirectory = FileManager.default.currentDirectoryPath
+ guard FileManager.default.changeCurrentDirectoryPath(outDir) else {
+ throw Console.Error("Could not change directory to '\(outDir)'")
+ }
+ defer { FileManager.default.changeCurrentDirectoryPath(oldDirectory) }
+
+ let inputStream = DataReader.data(readingFrom: fd.rawValue)
+
+ let (observer, source) = inputStream.lockstepSplit()
+
+ async let readTask: Void = {
+ var read = 0
+ for try await chunk in observer {
+ read += chunk.count
+ let progress = Int(Double(read) / Double(length) * 100)
+ print("\r[Extracting XIP] \(progress)%", terminator: "")
+ fflush(stdoutSafe)
+ if read == length { break }
+ }
+ }()
+
+ let xipToChunks = XIP.transform(
+ DataReader(data: source),
+ options: nil
+ )
+
+ // ideally we would filter out the files we don't want at this stage,
+ // speeding up extraction AND entirely avoiding the "Installing SDKs"
+ // post-processing step. However, files in xip archives may be hardlinks
+ // to one another and so we need to handle the case where a file inside
+ // our filter() points to a file outside of it. We might be able to do this
+ // with a double-pass system.
+
+ let chunksToFiles = Chunks.transform(
+ xipToChunks,
+ options: nil
+ )
+
+ let filesToDisk = Files.transform(
+ chunksToFiles,
+ options: .init(
+ compress: false,
+ dryRun: false
+ )
+ )
+
+ var wanted = 0
+ for try await file in filesToDisk {
+ wanted += Self.isWanted(file.name[...]) ? 1 : 0
+ }
+ _ = try await readTask
+
+ print()
+
+ return wanted
+ }
+
+ private static func isWanted(_ path: Substring) -> Bool {
+ var components = path.split(separator: "/")[...]
+ if components.first == "." {
+ components.removeFirst()
+ }
+ if components.first?.hasSuffix(".app") == true {
+ components.removeFirst()
+ }
+ guard SDKEntry.wanted.matches(components) else { return false }
+
+ // TODO: see if we can exclude most dylibs. Seems we need the XCTest ones though.
+
+ if components.count >= 10 && components[components.startIndex + 9] == "prebuilt-modules" && components.starts(
+ with: "Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift"
+ .split(separator: "/")
+ ) {
+ return false
+ }
+
+ return true
+ }
+}
+
+struct SDKDefinition: Encodable {
+ struct Triple: Encodable {
+ var sdkRootPath: String
+ var includeSearchPaths: [String]
+ var librarySearchPaths: [String]
+ var swiftResourcesPath: String
+ var swiftStaticResourcesPath: String
+ var toolsetPaths: [String]
+ }
+ var schemaVersion: String
+ var targetTriples: [String: Triple]
+}
+
+struct SDKEntry {
+ var names: Set<Substring>
+ var values: [SDKEntry] = []
+
+ // empty = wildcard
+ init(_ names: Set<Substring>, _ values: [SDKEntry] = []) {
+ self.names = names
+ self.values = values
+ }
+
+ init(_ name: Substring, _ values: [SDKEntry] = []) {
+ self.init([name], values)
+ }
+
+ func matches(_ path: ArraySlice<Substring>) -> Bool {
+ guard let first = path.first else { return true }
+ guard names.isEmpty || names.contains(first) else { return false }
+ if values.isEmpty { return true } // leaf, everything after is good
+ let afterName = path.dropFirst()
+ for value in values {
+ if value.matches(afterName) {
+ return true
+ }
+ }
+ return false
+ }
+
+ static func E(_ name: Substring?, _ values: [SDKEntry] = []) -> SDKEntry {
+ guard let name else { return SDKEntry([], values) }
+ let parts = name.split(separator: "/").reversed()
+ return parts.dropFirst().reduce(SDKEntry(parts.first!, values)) { SDKEntry($1, [$0]) }
+ }
+
+ static let wanted = E("Contents/Developer", [
+ E("Toolchains/XcodeDefault.xctoolchain/usr/lib", [
+ E("swift"),
+ E("swift_static"),
+ E("clang"),
+ ]),
+ E("Platforms", ["iPhoneOS", "MacOSX", "iPhoneSimulator"].map {
+ E("\($0).platform/Developer", [
+ E("SDKs"),
+ E("Library", [
+ E("Frameworks"),
+ E("PrivateFrameworks"),
+ ]),
+ E("usr/lib"),
+ ])
+ }),
+ ])
+}
diff --git a/Tests/SupersignCLITests/MatcherTests.swift b/Tests/SupersignCLITests/MatcherTests.swift
new file mode 100644
index 0000000..903e9a8
--- /dev/null
+++ b/Tests/SupersignCLITests/MatcherTests.swift
@@ -0,0 +1,52 @@
+import Testing
+@testable import SupersignCLISupport
+
+extension SDKEntry {
+ fileprivate static let test = E("Root", [
+ E("Intermediate", [
+ E("Leaf"),
+ ]),
+ E("AnotherLeaf"),
+ E("BeforeWildcard", [
+ E(nil, [
+ E("AfterWildcard"),
+ ]),
+ ]),
+ ])
+}
+
+@Test func testMatcherSupportsSlices() {
+ let s = SDKEntry.test
+
+ let slice0 = ["Root"].map { $0[...] }[0...]
+ #expect(s.matches(slice0))
+
+ let slice1 = ["foo", "bar", "Root"].map { $0[...] }[2...]
+ #expect(s.matches(slice1))
+
+ let slice2 = ["foo", "bar", "NotRoot"].map { $0[...] }[2...]
+ #expect(!s.matches(slice2))
+}
+
+@Test func testMatcherLeaves() {
+ let s = SDKEntry.test
+
+ #expect(s.matches(["Root", "Intermediate"]))
+ #expect(!s.matches(["Root", "NotIntermediate"]))
+
+ #expect(s.matches(["Root", "Intermediate", "Leaf"]))
+ #expect(!s.matches(["Root", "Intermediate", "NotLeaf"]))
+ #expect(s.matches(["Root", "Intermediate", "Leaf", "AfterLeaf"]))
+
+ #expect(s.matches(["Root", "AnotherLeaf"]))
+ #expect(s.matches(["Root", "AnotherLeaf", "AfterAnotherLeaf"]))
+
+ #expect(!s.matches(["Junk", "Root"]))
+
+ #expect(s.matches(["Root", "BeforeWildcard"]))
+ #expect(s.matches(["Root", "BeforeWildcard", "Blah"]))
+ #expect(s.matches(["Root", "BeforeWildcard", "Blah", "AfterWildcard"]))
+ #expect(s.matches(["Root", "BeforeWildcard", "Blah", "AfterWildcard", "Suffix"]))
+ #expect(!s.matches(["Root", "BeforeWildcard", "Blah", "NotAfterWildcard"]))
+ #expect(!s.matches(["Root", "BeforeWildcard", "Blah", "NotAfterWildcard", "Suffix"]))
+}