Cross-platform Xcode replacement. Build and deploy iOS apps with SwiftPM on Linux, Windows, macOS.
SDK builder initial pass
Kabir Oberaibaf71fa1da6e
diff --git a/Dockerfile b/Dockerfileindex 046b74a..b3f51e2 100644--- a/Dockerfile+++ b/Dockerfile@@ -1,7 +1,7 @@# Note: We use 20.04 since AppImage recommends building on the# oldest configuration that you support-FROM swift:6.0-focal AS build-base+FROM swift:6.1-focal AS build-baseRUN apt-get update \&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \@@ -17,6 +17,7 @@ RUN apt-get update \libxml2 \curl libcurl4-openssl-dev \zip unzip \+ liblzma-dev zlib1g-dev \&& rm -rf /var/lib/apt/lists/*diff --git a/Package.resolved b/Package.resolvedindex 8654e97..a70ee8d 100644--- a/Package.resolved+++ b/Package.resolved@@ -1,5 +1,5 @@{- "originHash" : "360bc3ae9f3e8613a4fb4b6583a90d4819977ed68df921d928f49717dad97299",+ "originHash" : "7d8342a181303ec948ba0a80e3bdadfdefbde5ca30e750282a9caf834410fae7","pins" : [{"identity" : "aexml",@@ -334,6 +334,15 @@"version" : "1.3.1"}},+ {+ "identity" : "unxip",+ "kind" : "remoteSourceControl",+ "location" : "https://github.com/saagarjha/unxip",+ "state" : {+ "revision" : "d6f0009109f7502f8f5b3f23dcd5b43cf5b29b32",+ "version" : "3.1.0"+ }+ },{"identity" : "version","kind" : "remoteSourceControl",diff --git a/Package.swift b/Package.swiftindex 1e3c459..3fe0c05 100644--- a/Package.swift+++ b/Package.swift@@ -51,6 +51,7 @@ let package = Package(.package(url: "https://github.com/attaswift/BigInt", from: "5.5.0"),.package(url: "https://github.com/mxcl/Version", from: "2.1.0"),.package(url: "https://github.com/jpsim/Yams", from: "5.1.3"),+ .package(url: "https://github.com/saagarjha/unxip", from: "3.1.0"),// TODO: just depend on tuist/XcodeProj instead.package(url: "https://github.com/yonaskolb/XcodeGen", from: "2.43.0"),@@ -109,6 +110,12 @@ let package = Package(],cSettings: cSettings),+ .testTarget(+ name: "SupersignCLITests",+ dependencies: [+ "SupersignCLISupport",+ ]+ ),.testTarget(name: "SupersignTests",dependencies: [@@ -133,6 +140,7 @@ let package = Package(.product(name: "SystemPackage", package: "swift-system"),.product(name: "NIOPosix", package: "swift-nio"),.product(name: "Version", package: "Version"),+ .product(name: "libunxip", package: "unxip"),],cSettings: cSettings),diff --git a/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swift b/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swiftindex 78a51d3..4e59a87 100644--- a/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swift+++ b/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swift@@ -12,11 +12,69 @@ struct DevSDKCommand: AsyncParsableCommand {subcommands: [DevSDKInstallCommand.self,DevSDKRemoveCommand.self,+ DevSDKBuildCommand.self,],defaultSubcommand: DevSDKInstallCommand.self)}+struct DevSDKBuildCommand: AsyncParsableCommand {+ static let configuration = CommandConfiguration(+ commandName: "build",+ abstract: "Build the Darwin SDK from Xcode.xip"+ )++ @Argument(+ help: "Path to Xcode.xip or Xcode.app",+ completion: .file(extensions: ["xip", "app"])+ )+ var path: String++ @Argument(+ help: "Output directory"+ )+ var outputDir: String++ @Option(+ help: ArgumentHelp(+ "The architecture of the Linux host the SDK is being built for.",+ discussion: "Defaults to 'auto', which attempts to match the current host architecture."+ )+ ) var arch: ArchSelection = .auto++ func run() async throws {+ let builderArch: SDKBuilder.Arch = switch arch {+ case .auto:+ #if arch(arm64)+ .aarch64+ #elseif arch(x86_64)+ .x86_64+ #else+ throw Console.Error("Could not auto-detect target architecture. Please specify one with '--arch'.")+ #endif+ case .arm64: .aarch64+ case .x86_64: .x86_64+ }++ let input: SDKBuilder.Input = if path.hasSuffix(".xip") {+ .xip(path)+ } else if path.hasSuffix(".app") || path.hasSuffix(".app/") {+ .app(path)+ } else {+ throw Console.Error("Expected input path to end in .xip or .app")+ }++ let builder = SDKBuilder(input: input, outputPath: outputDir, arch: builderArch)+ try await builder.buildSDK()+ }+}++enum ArchSelection: String, ExpressibleByArgument {+ case auto+ case x86_64+ case arm64+}+struct DevSDKInstallCommand: AsyncParsableCommand {static let configuration = CommandConfiguration(commandName: "install",diff --git a/Sources/SupersignCLISupport/DevCommands/SDKBuilder.swift b/Sources/SupersignCLISupport/DevCommands/SDKBuilder.swiftnew file mode 100644index 0000000..3ad57ef--- /dev/null+++ b/Sources/SupersignCLISupport/DevCommands/SDKBuilder.swift@@ -0,0 +1,441 @@+import Foundation+import Dependencies+import SystemPackage+import libunxip+import Supersign // HTTPClient, stdoutSafe+import PackLib // ToolRegistry++struct SDKBuilder {+ enum Arch: String {+ case x86_64+ case aarch64+ }++ enum Input {+ case xip(String)+ case app(String)+ }++ let input: Input+ let outputPath: String+ let arch: Arch++ func buildSDK() async throws {+ // tag from https://github.com/kabiroberai/darwin-tools-linux-llvm+ let darwinToolsVersion = "1.0.1"++ // TODO: store relevant info for staleness check+ let sdkVersion = "develop"++ print("[Cleaning stage]")++ let output = URL(fileURLWithPath: outputPath, isDirectory: true)+ .appendingPathComponent("darwin.artifactbundle")++ try? FileManager.default.removeItem(at: output)+ try FileManager.default.createDirectory(+ at: output,+ withIntermediateDirectories: true+ )++ let toolsetDir = output.appendingPathComponent("toolset")++ try FileManager.default.createDirectory(+ at: toolsetDir,+ withIntermediateDirectories: false+ )++ let pipe = Pipe()+ let untar = Process()+ untar.currentDirectoryURL = toolsetDir+ untar.executableURL = try await ToolRegistry.locate("tar")+ untar.arguments = ["xzf", "-"]+ untar.standardInput = pipe.fileHandleForReading+ async let tarExit: Void = untar.runUntilExit()++ @Dependency(\.httpClient) var httpClient+ let url = URL(string: """+ https://github.com/kabiroberai/darwin-tools-linux-llvm/releases/download/\+ v\(darwinToolsVersion)/toolset-\(arch.rawValue).tar.gz+ """)!+ let (response, body) = try await httpClient.send(HTTPRequest(url: url))+ guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") }+ let length: Int64? = switch body.length {+ case .known(let known): known+ case .unknown: nil+ }+ let writer = pipe.fileHandleForWriting+ var written: Int64 = 0+ for try await chunk in body {+ try writer.write(contentsOf: chunk)+ written += Int64(chunk.count)+ if let length {+ let progress = Int(Double(written) / Double(length) * 100)+ print("\r[Installing toolchain] \(progress)%", terminator: "")+ fflush(stdoutSafe)+ }+ }+ print()+ try writer.close()+ try await tarExit++ let dev = output.appendingPathComponent("Developer")++ let appDir: URL+ let cleanupStageDir: URL?+ let wanted: Int?++ switch input {+ case .xip(let inputPath):+ let devStage = output.appendingPathComponent("DeveloperStage")+ try FileManager.default.createDirectory(at: devStage, withIntermediateDirectories: false)+ wanted = try await extractXIP(inputPath: inputPath, outDir: devStage.path)+ appDir = devStage.appendingPathComponent("Xcode.app")+ cleanupStageDir = devStage+ case .app(let appPath):+ wanted = nil+ appDir = URL(fileURLWithPath: appPath)+ cleanupStageDir = nil+ }++ try FileManager.default.createDirectory(at: dev, withIntermediateDirectories: false)++ var toDoDirs: [String] = ["Contents/Developer"]+ var count = 0+ while let next = toDoDirs.popLast() {+ try Task.checkCancellation()++ let url = appDir.appendingPathComponent(next)+ let contents = try FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: [.isDirectoryKey])+ for child in contents {+ let path = "\(next)/\(child.lastPathComponent)"+ guard Self.isWanted(path[...]) else { continue }++ count += 1+ if let wanted {+ let progress = Int(Double(count) / Double(wanted) * 100)+ print("\r[Installing SDKs] \(progress)%", terminator: "")+ fflush(stdoutSafe)+ }+ if count % 100 == 0 {+ if wanted == nil {+ print("\r[Installing SDKs] Copied \(count) files", terminator: "")+ fflush(stdoutSafe)+ }+ await Task.yield()+ }++ let insideDeveloper = path.dropFirst("Contents/Developer/".count)+ guard !insideDeveloper.isEmpty else { continue }+ let dest = dev.appendingPathComponent(String(insideDeveloper))++ if try child.resourceValues(forKeys: [.isDirectoryKey]).isDirectory == true {+ toDoDirs.append(path)+ try FileManager.default.createDirectory(at: dest, withIntermediateDirectories: false)+ } else {+ try FileManager.default.copyItem(at: child, to: dest)+ }+ }+ }++ if wanted != nil {+ // otherwise the last log might say 99%+ print("\r[Installing SDKs] 100%", terminator: "")+ }+ print()++ print("[Cleaning XIP]")+ if let cleanupStageDir {+ try? FileManager.default.removeItem(at: cleanupStageDir)+ }++ func sdk(platform: String, prefix: String) throws -> String {+ let regex = try NSRegularExpression(pattern: #"^\#(prefix)\d+\.\d+\.sdk$"#)+ let dir = dev.appendingPathComponent("Platforms/\(platform).platform/Developer/SDKs")+ let names = try dir.contents().map(\.lastPathComponent)+ guard let name = names.first(where: {+ regex.firstMatch(in: $0, range: NSRange($0.startIndex..., in: $0)) != nil+ }) else {+ throw Console.Error("Could not find SDK for \(platform)/\(prefix)")+ }+ return name+ }++ let iPhoneOSSDK = try sdk(platform: "iPhoneOS", prefix: "iPhoneOS")+ let iPhoneSimSDK = try sdk(platform: "iPhoneSimulator", prefix: "iPhoneSimulator")+ let macOSSDK = try sdk(platform: "MacOSX", prefix: "MacOSX")++ print("""+ [Finalizing SDKs]+ - \(iPhoneOSSDK)+ - \(iPhoneSimSDK)+ - \(macOSSDK)+ """)++ /*+ XCTest and Testing.framework are located in *.platform/Developer/{Library/Frameworks,usr/lib} rather than inside+ the SDK. These search paths are explicitly included when building tests, which presumably ensures that normal+ applications don't accidentally link against them in production.++ SwiftPM makes no such affordances outside of macOS, so we add the usr/lib path as include/library search paths+ in the SDK config, and symlink the frameworks into the SDKs (since there's no frameworkSearchPaths option).+ While this drops a safeguard it's better than not having the testing libs at all.+ */++ for platform in ["iPhoneOS", "MacOSX", "iPhoneSimulator"] {+ let lib = "../../../../../Library"+ let dest = dev.appendingPathComponent("""+ Platforms/\(platform).platform/Developer/SDKs/\(platform).sdk\+ /System/Library/Frameworks+ """).path++ try FileManager.default.createSymbolicLink(+ atPath: "\(dest)/Testing.framework",+ withDestinationPath: "\(lib)/Frameworks/Testing.framework"+ )++ try FileManager.default.createSymbolicLink(+ atPath: "\(dest)/XCTest.framework",+ withDestinationPath: "\(lib)/Frameworks/XCTest.framework"+ )++ try FileManager.default.createSymbolicLink(+ atPath: "\(dest)/XCUIAutomation.framework",+ withDestinationPath: "\(lib)/Frameworks/XCUIAutomation.framework"+ )++ try FileManager.default.createSymbolicLink(+ atPath: "\(dest)/XCTestCore.framework",+ withDestinationPath: "\(lib)/PrivateFrameworks/XCTestCore.framework"+ )+ }++ print("[Writing metadata]")+ try """+ {+ "schemaVersion": "1.0",+ "artifacts": {+ "darwin": {+ "type": "swiftSDK",+ "version": "0.0.1",+ "variants": [+ {+ "path": ".",+ "supportedTriples": ["aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]+ }+ ]+ }+ }+ }+ """.write(+ to: output.appendingPathComponent("info.json"),+ atomically: false,+ encoding: .utf8+ )++ try """+ {+ "schemaVersion": "1.0",+ "rootPath": "toolset/bin",+ "linker": {+ "path": "ld64.lld"+ },+ "swiftCompiler": {+ "extraCLIOptions": [+ "-use-ld=lld"+ ]+ }+ }+ """.write(+ to: output.appendingPathComponent("toolset.json"),+ atomically: false,+ encoding: .utf8+ )++ func triple(platform: String, sdk: String) -> SDKDefinition.Triple {+ SDKDefinition.Triple(+ sdkRootPath: "Developer/Platforms/\(platform).platform/Developer/SDKs/\(sdk)",+ includeSearchPaths: ["Developer/Platforms/\(platform).platform/Developer/usr/lib"],+ librarySearchPaths: ["Developer/Platforms/\(platform).platform/Developer/usr/lib"],+ swiftResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift",+ swiftStaticResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static",+ toolsetPaths: ["toolset.json"]+ )+ }++ let sdkDefinition = SDKDefinition(+ schemaVersion: "4.0",+ targetTriples: [+ "arm64-apple-ios": triple(platform: "iPhoneOS", sdk: iPhoneOSSDK),+ "arm64-apple-ios-simulator": triple(platform: "iPhoneSimulator", sdk: iPhoneSimSDK),+ "x86_64-apple-ios-simulator": triple(platform: "iPhoneSimulator", sdk: iPhoneSimSDK),+ "arm64-apple-macosx": triple(platform: "MacOSX", sdk: macOSSDK),+ "x86_64-apple-macosx": triple(platform: "MacOSX", sdk: macOSSDK),+ ]+ )++ let encoder = JSONEncoder()+ try encoder+ .encode(sdkDefinition)+ .write(to: output.appendingPathComponent("swift-sdk.json"))++ try Data("\(sdkVersion)\n".utf8)+ .write(to: output.appendingPathComponent("darwin-sdk-version.txt"))++ print("Built SDK at \(output.path)")+ }++ // returns the number of files we actually want to keep,+ // useful for computing progress % during fs traversal+ private func extractXIP(inputPath: String, outDir: String) async throws -> Int {+ let fd = try FileDescriptor.open(inputPath, .readOnly)+ defer { try? fd.close() }++ let length = try fd.seek(offset: 0, from: .end)+ try fd.seek(offset: 0, from: .start)++ // global state, ah well+ let oldDirectory = FileManager.default.currentDirectoryPath+ guard FileManager.default.changeCurrentDirectoryPath(outDir) else {+ throw Console.Error("Could not change directory to '\(outDir)'")+ }+ defer { FileManager.default.changeCurrentDirectoryPath(oldDirectory) }++ let inputStream = DataReader.data(readingFrom: fd.rawValue)++ let (observer, source) = inputStream.lockstepSplit()++ async let readTask: Void = {+ var read = 0+ for try await chunk in observer {+ read += chunk.count+ let progress = Int(Double(read) / Double(length) * 100)+ print("\r[Extracting XIP] \(progress)%", terminator: "")+ fflush(stdoutSafe)+ if read == length { break }+ }+ }()++ let xipToChunks = XIP.transform(+ DataReader(data: source),+ options: nil+ )++ // ideally we would filter out the files we don't want at this stage,+ // speeding up extraction AND entirely avoiding the "Installing SDKs"+ // post-processing step. However, files in xip archives may be hardlinks+ // to one another and so we need to handle the case where a file inside+ // our filter() points to a file outside of it. We might be able to do this+ // with a double-pass system.++ let chunksToFiles = Chunks.transform(+ xipToChunks,+ options: nil+ )++ let filesToDisk = Files.transform(+ chunksToFiles,+ options: .init(+ compress: false,+ dryRun: false+ )+ )++ var wanted = 0+ for try await file in filesToDisk {+ wanted += Self.isWanted(file.name[...]) ? 1 : 0+ }+ _ = try await readTask++ print()++ return wanted+ }++ private static func isWanted(_ path: Substring) -> Bool {+ var components = path.split(separator: "/")[...]+ if components.first == "." {+ components.removeFirst()+ }+ if components.first?.hasSuffix(".app") == true {+ components.removeFirst()+ }+ guard SDKEntry.wanted.matches(components) else { return false }++ // TODO: see if we can exclude most dylibs. Seems we need the XCTest ones though.++ if components.count >= 10 && components[components.startIndex + 9] == "prebuilt-modules" && components.starts(+ with: "Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift"+ .split(separator: "/")+ ) {+ return false+ }++ return true+ }+}++struct SDKDefinition: Encodable {+ struct Triple: Encodable {+ var sdkRootPath: String+ var includeSearchPaths: [String]+ var librarySearchPaths: [String]+ var swiftResourcesPath: String+ var swiftStaticResourcesPath: String+ var toolsetPaths: [String]+ }+ var schemaVersion: String+ var targetTriples: [String: Triple]+}++struct SDKEntry {+ var names: Set<Substring>+ var values: [SDKEntry] = []++ // empty = wildcard+ init(_ names: Set<Substring>, _ values: [SDKEntry] = []) {+ self.names = names+ self.values = values+ }++ init(_ name: Substring, _ values: [SDKEntry] = []) {+ self.init([name], values)+ }++ func matches(_ path: ArraySlice<Substring>) -> Bool {+ guard let first = path.first else { return true }+ guard names.isEmpty || names.contains(first) else { return false }+ if values.isEmpty { return true } // leaf, everything after is good+ let afterName = path.dropFirst()+ for value in values {+ if value.matches(afterName) {+ return true+ }+ }+ return false+ }++ static func E(_ name: Substring?, _ values: [SDKEntry] = []) -> SDKEntry {+ guard let name else { return SDKEntry([], values) }+ let parts = name.split(separator: "/").reversed()+ return parts.dropFirst().reduce(SDKEntry(parts.first!, values)) { SDKEntry($1, [$0]) }+ }++ static let wanted = E("Contents/Developer", [+ E("Toolchains/XcodeDefault.xctoolchain/usr/lib", [+ E("swift"),+ E("swift_static"),+ E("clang"),+ ]),+ E("Platforms", ["iPhoneOS", "MacOSX", "iPhoneSimulator"].map {+ E("\($0).platform/Developer", [+ E("SDKs"),+ E("Library", [+ E("Frameworks"),+ E("PrivateFrameworks"),+ ]),+ E("usr/lib"),+ ])+ }),+ ])+}diff --git a/Tests/SupersignCLITests/MatcherTests.swift b/Tests/SupersignCLITests/MatcherTests.swiftnew file mode 100644index 0000000..903e9a8--- /dev/null+++ b/Tests/SupersignCLITests/MatcherTests.swift@@ -0,0 +1,52 @@+import Testing+@testable import SupersignCLISupport++extension SDKEntry {+ fileprivate static let test = E("Root", [+ E("Intermediate", [+ E("Leaf"),+ ]),+ E("AnotherLeaf"),+ E("BeforeWildcard", [+ E(nil, [+ E("AfterWildcard"),+ ]),+ ]),+ ])+}++@Test func testMatcherSupportsSlices() {+ let s = SDKEntry.test++ let slice0 = ["Root"].map { $0[...] }[0...]+ #expect(s.matches(slice0))++ let slice1 = ["foo", "bar", "Root"].map { $0[...] }[2...]+ #expect(s.matches(slice1))++ let slice2 = ["foo", "bar", "NotRoot"].map { $0[...] }[2...]+ #expect(!s.matches(slice2))+}++@Test func testMatcherLeaves() {+ let s = SDKEntry.test++ #expect(s.matches(["Root", "Intermediate"]))+ #expect(!s.matches(["Root", "NotIntermediate"]))++ #expect(s.matches(["Root", "Intermediate", "Leaf"]))+ #expect(!s.matches(["Root", "Intermediate", "NotLeaf"]))+ #expect(s.matches(["Root", "Intermediate", "Leaf", "AfterLeaf"]))++ #expect(s.matches(["Root", "AnotherLeaf"]))+ #expect(s.matches(["Root", "AnotherLeaf", "AfterAnotherLeaf"]))++ #expect(!s.matches(["Junk", "Root"]))++ #expect(s.matches(["Root", "BeforeWildcard"]))+ #expect(s.matches(["Root", "BeforeWildcard", "Blah"]))+ #expect(s.matches(["Root", "BeforeWildcard", "Blah", "AfterWildcard"]))+ #expect(s.matches(["Root", "BeforeWildcard", "Blah", "AfterWildcard", "Suffix"]))+ #expect(!s.matches(["Root", "BeforeWildcard", "Blah", "NotAfterWildcard"]))+ #expect(!s.matches(["Root", "BeforeWildcard", "Blah", "NotAfterWildcard", "Suffix"]))+}