From 317336a673beb3dcfd2bdf188deae9b65ecae15a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Fri, 6 Sep 2024 13:18:06 +0200 Subject: [PATCH 01/10] fix deprecation warning for Plugin's Path --- Plugins/AWSLambdaPackager/Plugin.swift | 222 ++++++-------------- Plugins/AWSLambdaPackager/PluginUtils.swift | 145 +++++++++++++ 2 files changed, 215 insertions(+), 152 deletions(-) create mode 100644 Plugins/AWSLambdaPackager/PluginUtils.swift diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index a0ea999b..fb0b51f0 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -12,11 +12,19 @@ // //===----------------------------------------------------------------------===// -import Dispatch -import Foundation +//import Dispatch import PackagePlugin import Synchronization +#if canImport(FoundationEssentials) +import FoundationEssentials +#else +import struct Foundation.URL +import class Foundation.ProcessInfo +import class Foundation.FileManager +import struct Foundation.ObjCBool +#endif + @available(macOS 15.0, *) @main struct AWSLambdaPackager: CommandPlugin { @@ -33,7 +41,7 @@ struct AWSLambdaPackager: CommandPlugin { ) } - let builtProducts: [LambdaProduct: Path] + let builtProducts: [LambdaProduct: URL] if self.isAmazonLinux2() { // build directly on the machine builtProducts = try self.build( @@ -46,9 +54,9 @@ struct AWSLambdaPackager: CommandPlugin { // build with docker builtProducts = try self.buildInDocker( packageIdentity: context.package.id, - packageDirectory: context.package.directory, + packageDirectory: context.package.directoryURL, products: configuration.products, - toolsProvider: { name in try context.tool(named: name).path }, + toolsProvider: { name in try context.tool(named: name).url }, outputDirectory: configuration.outputDirectory, baseImage: configuration.baseDockerImage, disableDockerImageUpdate: configuration.disableDockerImageUpdate, @@ -61,7 +69,7 @@ struct AWSLambdaPackager: CommandPlugin { let archives = try self.package( packageName: context.package.displayName, products: builtProducts, - toolsProvider: { name in try context.tool(named: name).path }, + toolsProvider: { name in try context.tool(named: name).url }, outputDirectory: configuration.outputDirectory, verboseLogging: configuration.verboseLogging ) @@ -70,21 +78,21 @@ struct AWSLambdaPackager: CommandPlugin { "\(archives.count > 0 ? archives.count.description : "no") archive\(archives.count != 1 ? "s" : "") created" ) for (product, archivePath) in archives { - print(" * \(product.name) at \(archivePath.string)") + print(" * \(product.name) at \(archivePath)") } } private func buildInDocker( packageIdentity: Package.ID, - packageDirectory: Path, + packageDirectory: URL, products: [Product], - toolsProvider: (String) throws -> Path, - outputDirectory: Path, + toolsProvider: (String) throws -> URL, + outputDirectory: URL, baseImage: String, disableDockerImageUpdate: Bool, buildConfiguration: PackageManager.BuildConfiguration, verboseLogging: Bool - ) throws -> [LambdaProduct: Path] { + ) throws -> [LambdaProduct: URL] { let dockerToolPath = try toolsProvider("docker") print("-------------------------------------------------------------------------") @@ -94,7 +102,7 @@ struct AWSLambdaPackager: CommandPlugin { if !disableDockerImageUpdate { // update the underlying docker image, if necessary print("updating \"\(baseImage)\" docker image") - try self.execute( + try Utils.execute( executable: dockerToolPath, arguments: ["pull", baseImage], logLevel: .output @@ -103,10 +111,10 @@ struct AWSLambdaPackager: CommandPlugin { // get the build output path let buildOutputPathCommand = "swift build -c \(buildConfiguration.rawValue) --show-bin-path" - let dockerBuildOutputPath = try self.execute( + let dockerBuildOutputPath = try Utils.execute( executable: dockerToolPath, arguments: [ - "run", "--rm", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", baseImage, "bash", + "run", "--rm", "-v", "\(packageDirectory.path()):/workspace", "-w", "/workspace", baseImage, "bash", "-cl", buildOutputPathCommand, ], logLevel: verboseLogging ? .debug : .silent @@ -114,12 +122,10 @@ struct AWSLambdaPackager: CommandPlugin { guard let buildPathOutput = dockerBuildOutputPath.split(separator: "\n").last else { throw Errors.failedParsingDockerOutput(dockerBuildOutputPath) } - let buildOutputPath = Path( - buildPathOutput.replacingOccurrences(of: "/workspace", with: packageDirectory.string) - ) + let buildOutputPath = URL(string: buildPathOutput.replacingOccurrences(of: "/workspace/", with: packageDirectory.description))! // build the products - var builtProducts = [LambdaProduct: Path]() + var builtProducts = [LambdaProduct: URL]() for product in products { print("building \"\(product.name)\"") let buildCommand = @@ -128,30 +134,32 @@ struct AWSLambdaPackager: CommandPlugin { // when developing locally, we must have the full swift-aws-lambda-runtime project in the container // because Examples' Package.swift have a dependency on ../.. // just like Package.swift's examples assume ../.., we assume we are two levels below the root project - let lastComponent = packageDirectory.lastComponent - let beforeLastComponent = packageDirectory.removingLastComponent().lastComponent - try self.execute( + let slice = packageDirectory.pathComponents.suffix(2) + let beforeLastComponent = packageDirectory.pathComponents[slice.startIndex] + let lastComponent = packageDirectory.pathComponents[slice.endIndex-1] + try Utils.execute( executable: dockerToolPath, arguments: [ "run", "--rm", "--env", "LAMBDA_USE_LOCAL_DEPS=true", "-v", - "\(packageDirectory.string)/../..:/workspace", "-w", + "\(packageDirectory.path())../..:/workspace", "-w", "/workspace/\(beforeLastComponent)/\(lastComponent)", baseImage, "bash", "-cl", buildCommand, ], logLevel: verboseLogging ? .debug : .output ) } else { - try self.execute( + try Utils.execute( executable: dockerToolPath, arguments: [ - "run", "--rm", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", baseImage, + "run", "--rm", "-v", "\(packageDirectory.path()):/workspace", "-w", "/workspace", baseImage, "bash", "-cl", buildCommand, ], logLevel: verboseLogging ? .debug : .output ) } - let productPath = buildOutputPath.appending(product.name) - guard FileManager.default.fileExists(atPath: productPath.string) else { - Diagnostics.error("expected '\(product.name)' binary at \"\(productPath.string)\"") + let productPath = buildOutputPath.appending(path: product.name) + + guard FileManager.default.fileExists(atPath: productPath.path()) else { + Diagnostics.error("expected '\(product.name)' binary at \"\(productPath.path())\"") throw Errors.productExecutableNotFound(product.name) } builtProducts[.init(product)] = productPath @@ -164,12 +172,12 @@ struct AWSLambdaPackager: CommandPlugin { products: [Product], buildConfiguration: PackageManager.BuildConfiguration, verboseLogging: Bool - ) throws -> [LambdaProduct: Path] { + ) throws -> [LambdaProduct: URL] { print("-------------------------------------------------------------------------") print("building \"\(packageIdentity)\"") print("-------------------------------------------------------------------------") - var results = [LambdaProduct: Path]() + var results = [LambdaProduct: URL]() for product in products { print("building \"\(product.name)\"") var parameters = PackageManager.BuildParameters() @@ -184,7 +192,7 @@ struct AWSLambdaPackager: CommandPlugin { guard let artifact = result.executableArtifact(for: product) else { throw Errors.productExecutableNotFound(product.name) } - results[.init(product)] = artifact.path + results[.init(product)] = artifact.url } return results } @@ -192,34 +200,34 @@ struct AWSLambdaPackager: CommandPlugin { // TODO: explore using ziplib or similar instead of shelling out private func package( packageName: String, - products: [LambdaProduct: Path], - toolsProvider: (String) throws -> Path, - outputDirectory: Path, + products: [LambdaProduct: URL], + toolsProvider: (String) throws -> URL, + outputDirectory: URL, verboseLogging: Bool - ) throws -> [LambdaProduct: Path] { + ) throws -> [LambdaProduct: URL] { let zipToolPath = try toolsProvider("zip") - var archives = [LambdaProduct: Path]() + var archives = [LambdaProduct: URL]() for (product, artifactPath) in products { print("-------------------------------------------------------------------------") print("archiving \"\(product.name)\"") print("-------------------------------------------------------------------------") // prep zipfile location - let workingDirectory = outputDirectory.appending(product.name) - let zipfilePath = workingDirectory.appending("\(product.name).zip") - if FileManager.default.fileExists(atPath: workingDirectory.string) { - try FileManager.default.removeItem(atPath: workingDirectory.string) + let workingDirectory = outputDirectory.appending(path: product.name) + let zipfilePath = workingDirectory.appending(path: "\(product.name).zip") + if FileManager.default.fileExists(atPath: workingDirectory.path()) { + try FileManager.default.removeItem(atPath: workingDirectory.path()) } - try FileManager.default.createDirectory(atPath: workingDirectory.string, withIntermediateDirectories: true) + try FileManager.default.createDirectory(atPath: workingDirectory.path(), withIntermediateDirectories: true) // rename artifact to "bootstrap" - let relocatedArtifactPath = workingDirectory.appending(artifactPath.lastComponent) - let symbolicLinkPath = workingDirectory.appending("bootstrap") - try FileManager.default.copyItem(atPath: artifactPath.string, toPath: relocatedArtifactPath.string) + let relocatedArtifactPath = workingDirectory.appending(path: artifactPath.lastPathComponent) + let symbolicLinkPath = workingDirectory.appending(path: "bootstrap") + try FileManager.default.copyItem(atPath: artifactPath.path(), toPath: relocatedArtifactPath.path()) try FileManager.default.createSymbolicLink( - atPath: symbolicLinkPath.string, - withDestinationPath: relocatedArtifactPath.lastComponent + atPath: symbolicLinkPath.path(), + withDestinationPath: relocatedArtifactPath.lastPathComponent ) var arguments: [String] = [] @@ -227,29 +235,33 @@ struct AWSLambdaPackager: CommandPlugin { arguments = [ "--recurse-paths", "--symlinks", - zipfilePath.lastComponent, - relocatedArtifactPath.lastComponent, - symbolicLinkPath.lastComponent, + zipfilePath.lastPathComponent, + relocatedArtifactPath.lastPathComponent, + symbolicLinkPath.lastPathComponent, ] #else throw Errors.unsupportedPlatform("can't or don't know how to create a zip file on this platform") #endif // add resources - let artifactDirectory = artifactPath.removingLastComponent() + var artifactPathComponents = artifactPath.pathComponents + _ = artifactPathComponents.removeLast() + let artifactDirectory = artifactPathComponents.joined(separator: "/") let resourcesDirectoryName = "\(packageName)_\(product.name).resources" let resourcesDirectory = artifactDirectory.appending(resourcesDirectoryName) - let relocatedResourcesDirectory = workingDirectory.appending(resourcesDirectoryName) - if FileManager.default.fileExists(atPath: resourcesDirectory.string) { + let relocatedResourcesDirectory = workingDirectory.appending(path: resourcesDirectoryName) + print("--------- resources ----------") + if FileManager.default.fileExists(atPath: resourcesDirectory) { + print("--------- copying resources ----------") try FileManager.default.copyItem( - atPath: resourcesDirectory.string, - toPath: relocatedResourcesDirectory.string + atPath: resourcesDirectory, + toPath: relocatedResourcesDirectory.path() ) arguments.append(resourcesDirectoryName) } // run the zip tool - try self.execute( + try Utils.execute( executable: zipToolPath, arguments: arguments, customWorkingDirectory: workingDirectory, @@ -261,100 +273,6 @@ struct AWSLambdaPackager: CommandPlugin { return archives } - @discardableResult - private func execute( - executable: Path, - arguments: [String], - customWorkingDirectory: Path? = .none, - logLevel: ProcessLogLevel - ) throws -> String { - if logLevel >= .debug { - print("\(executable.string) \(arguments.joined(separator: " "))") - } - - let fd = dup(1) - let stdout = fdopen(fd, "rw")! - defer { fclose(stdout) } - - // We need to use an unsafe transfer here to get the fd into our Sendable closure. - // This transfer is fine, because we guarantee that the code in the outputHandler - // is run before we continue the functions execution, where the fd is used again. - // See `process.waitUntilExit()` and the following `outputSync.wait()` - struct UnsafeTransfer: @unchecked Sendable { - let value: Value - } - - let outputMutex = Mutex("") - let outputSync = DispatchGroup() - let outputQueue = DispatchQueue(label: "AWSLambdaPackager.output") - let unsafeTransfer = UnsafeTransfer(value: stdout) - let outputHandler = { @Sendable (data: Data?) in - dispatchPrecondition(condition: .onQueue(outputQueue)) - - outputSync.enter() - defer { outputSync.leave() } - - guard - let _output = data.flatMap({ - String(data: $0, encoding: .utf8)?.trimmingCharacters(in: CharacterSet(["\n"])) - }), !_output.isEmpty - else { - return - } - - outputMutex.withLock { output in - output += _output + "\n" - } - - switch logLevel { - case .silent: - break - case .debug(let outputIndent), .output(let outputIndent): - print(String(repeating: " ", count: outputIndent), terminator: "") - print(_output) - fflush(unsafeTransfer.value) - } - } - - let pipe = Pipe() - pipe.fileHandleForReading.readabilityHandler = { fileHandle in - outputQueue.async { outputHandler(fileHandle.availableData) } - } - - let process = Process() - process.standardOutput = pipe - process.standardError = pipe - process.executableURL = URL(fileURLWithPath: executable.string) - process.arguments = arguments - if let workingDirectory = customWorkingDirectory { - process.currentDirectoryURL = URL(fileURLWithPath: workingDirectory.string) - } - process.terminationHandler = { _ in - outputQueue.async { - outputHandler(try? pipe.fileHandleForReading.readToEnd()) - } - } - - try process.run() - process.waitUntilExit() - - // wait for output to be full processed - outputSync.wait() - - let output = outputMutex.withLock { $0 } - - if process.terminationStatus != 0 { - // print output on failure and if not already printed - if logLevel < .output { - print(output) - fflush(stdout) - } - throw Errors.processFailed([executable.string] + arguments, process.terminationStatus) - } - - return output - } - private func isAmazonLinux2() -> Bool { if let data = FileManager.default.contents(atPath: "/etc/system-release"), let release = String(data: data, encoding: .utf8) @@ -368,7 +286,7 @@ struct AWSLambdaPackager: CommandPlugin { @available(macOS 15.0, *) private struct Configuration: CustomStringConvertible { - public let outputDirectory: Path + public let outputDirectory: URL public let products: [Product] public let explicitProducts: Bool public let buildConfiguration: PackageManager.BuildConfiguration @@ -397,9 +315,9 @@ private struct Configuration: CustomStringConvertible { else { throw Errors.invalidArgument("invalid output directory '\(outputPath)'") } - self.outputDirectory = Path(outputPath) + self.outputDirectory = URL(string: outputPath)! } else { - self.outputDirectory = context.pluginWorkDirectory.appending(subpath: "\(AWSLambdaPackager.self)") + self.outputDirectory = context.pluginWorkDirectoryURL.appending(path: "\(AWSLambdaPackager.self)") } self.explicitProducts = !productsArgument.isEmpty @@ -537,7 +455,7 @@ private struct LambdaProduct: Hashable { extension PackageManager.BuildResult { // find the executable produced by the build func executableArtifact(for product: Product) -> PackageManager.BuildResult.BuiltArtifact? { - let executables = self.builtArtifacts.filter { $0.kind == .executable && $0.path.lastComponent == product.name } + let executables = self.builtArtifacts.filter { $0.kind == .executable && $0.url.lastPathComponent == product.name } guard !executables.isEmpty else { return nil } diff --git a/Plugins/AWSLambdaPackager/PluginUtils.swift b/Plugins/AWSLambdaPackager/PluginUtils.swift new file mode 100644 index 00000000..5742554c --- /dev/null +++ b/Plugins/AWSLambdaPackager/PluginUtils.swift @@ -0,0 +1,145 @@ +// ===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftAWSLambdaRuntime open source project +// +// Copyright (c) 2024 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftAWSLambdaRuntime project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +// ===----------------------------------------------------------------------===// + +import Dispatch +import PackagePlugin + +#if canImport(FoundationEssentials) +import FoundationEssentials +#else +import struct Foundation.URL +import struct Foundation.Data +import struct Foundation.CharacterSet +import class Foundation.Process +import class Foundation.Pipe +#endif + +struct Utils { + @discardableResult + static func execute( + executable: URL, + arguments: [String], + customWorkingDirectory: URL? = nil, + logLevel: ProcessLogLevel + ) throws -> String { + if logLevel >= .debug { + print("\(executable.absoluteString) \(arguments.joined(separator: " "))") + } + + // this shared global variable is safe because we're mutating it in a dispatch group + // https://developer.apple.com/documentation/foundation/process/1408746-terminationhandler + nonisolated(unsafe) var output = "" + let outputSync = DispatchGroup() + let outputQueue = DispatchQueue(label: "AWSLambdaPlugin.output") + let outputHandler = { @Sendable (data: Data?) in + dispatchPrecondition(condition: .onQueue(outputQueue)) + + outputSync.enter() + defer { outputSync.leave() } + + guard let _output = data.flatMap({ String(decoding: $0, as: UTF8.self).trimmingCharacters(in: CharacterSet(["\n"])) }), !_output.isEmpty else { + return + } + + output += _output + "\n" + + switch logLevel { + case .silent: + break + case .debug(let outputIndent), .output(let outputIndent): + print(String(repeating: " ", count: outputIndent), terminator: "") + print(_output) + fflush(stdout) + } + } + + let pipe = Pipe() + pipe.fileHandleForReading.readabilityHandler = { fileHandle in + outputQueue.async { + outputHandler(fileHandle.availableData) + } + } + + let process = Process() + process.standardOutput = pipe + process.standardError = pipe + process.executableURL = executable + process.arguments = arguments + if let customWorkingDirectory { + process.currentDirectoryURL = customWorkingDirectory + } + process.terminationHandler = { _ in + outputQueue.async { + outputHandler(try? pipe.fileHandleForReading.readToEnd()) + } + } + + try process.run() + process.waitUntilExit() + + // wait for output to be full processed + outputSync.wait() + + if process.terminationStatus != 0 { + // print output on failure and if not already printed + if logLevel < .output { + print(output) + fflush(stdout) + } + throw ProcessError.processFailed([executable.absoluteString] + arguments, process.terminationStatus, output) + } + + return output + } + + enum ProcessError: Error, CustomStringConvertible { + case processFailed([String], Int32, String) + + var description: String { + switch self { + case .processFailed(let arguments, let code, _): + return "\(arguments.joined(separator: " ")) failed with code \(code)" + } + } + } + + enum ProcessLogLevel: Comparable { + case silent + case output(outputIndent: Int) + case debug(outputIndent: Int) + + var naturalOrder: Int { + switch self { + case .silent: + return 0 + case .output: + return 1 + case .debug: + return 2 + } + } + + static var output: Self { + .output(outputIndent: 2) + } + + static var debug: Self { + .debug(outputIndent: 2) + } + + static func < (lhs: ProcessLogLevel, rhs: ProcessLogLevel) -> Bool { + lhs.naturalOrder < rhs.naturalOrder + } + } +} From 0256e6adda80fec39292a771888ca46a1f6729da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Fri, 6 Sep 2024 13:23:23 +0200 Subject: [PATCH 02/10] remove unused code --- Plugins/AWSLambdaPackager/Plugin.swift | 3 --- 1 file changed, 3 deletions(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index fb0b51f0..34f0b46f 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -//import Dispatch import PackagePlugin import Synchronization @@ -250,9 +249,7 @@ struct AWSLambdaPackager: CommandPlugin { let resourcesDirectoryName = "\(packageName)_\(product.name).resources" let resourcesDirectory = artifactDirectory.appending(resourcesDirectoryName) let relocatedResourcesDirectory = workingDirectory.appending(path: resourcesDirectoryName) - print("--------- resources ----------") if FileManager.default.fileExists(atPath: resourcesDirectory) { - print("--------- copying resources ----------") try FileManager.default.copyItem( atPath: resourcesDirectory, toPath: relocatedResourcesDirectory.path() From 6afd7caf7e6a15d0801f4ae3281f688a30d66958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Fri, 6 Sep 2024 13:52:12 +0200 Subject: [PATCH 03/10] remove deps on ObjCBool when compiling on Linux --- Plugins/AWSLambdaPackager/Plugin.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index 34f0b46f..ed8efb0c 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -307,8 +307,12 @@ private struct Configuration: CustomStringConvertible { self.verboseLogging = verboseArgument if let outputPath = outputPathArgument.first { + #if os(Linux) + var isDirectory: Bool = false + #else var isDirectory: ObjCBool = false - guard FileManager.default.fileExists(atPath: outputPath, isDirectory: &isDirectory), isDirectory.boolValue + #endif + guard FileManager.default.fileExists(atPath: outputPath, isDirectory: &isDirectory) else { throw Errors.invalidArgument("invalid output directory '\(outputPath)'") } From a79defc371f5fb5fc31d739065054e824c3e2bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Fri, 6 Sep 2024 13:58:58 +0200 Subject: [PATCH 04/10] fix license header --- Plugins/AWSLambdaPackager/PluginUtils.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Plugins/AWSLambdaPackager/PluginUtils.swift b/Plugins/AWSLambdaPackager/PluginUtils.swift index 5742554c..f71fac70 100644 --- a/Plugins/AWSLambdaPackager/PluginUtils.swift +++ b/Plugins/AWSLambdaPackager/PluginUtils.swift @@ -2,7 +2,7 @@ // // This source file is part of the SwiftAWSLambdaRuntime open source project // -// Copyright (c) 2024 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Copyright (c) 2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors // Licensed under Apache License v2.0 // // See LICENSE.txt for license information From b452c5af4843c70b2d9655262e69748d191b1e77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Fri, 6 Sep 2024 14:14:31 +0200 Subject: [PATCH 05/10] manually merge previous changes on `execute()` function --- Plugins/AWSLambdaPackager/PluginUtils.swift | 63 ++++++++++++--------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/Plugins/AWSLambdaPackager/PluginUtils.swift b/Plugins/AWSLambdaPackager/PluginUtils.swift index f71fac70..caa03326 100644 --- a/Plugins/AWSLambdaPackager/PluginUtils.swift +++ b/Plugins/AWSLambdaPackager/PluginUtils.swift @@ -14,45 +14,54 @@ import Dispatch import PackagePlugin - -#if canImport(FoundationEssentials) -import FoundationEssentials -#else -import struct Foundation.URL -import struct Foundation.Data -import struct Foundation.CharacterSet -import class Foundation.Process -import class Foundation.Pipe -#endif +import Synchronization +import Foundation struct Utils { @discardableResult static func execute( executable: URL, arguments: [String], - customWorkingDirectory: URL? = nil, + customWorkingDirectory: URL? = .none, logLevel: ProcessLogLevel ) throws -> String { if logLevel >= .debug { print("\(executable.absoluteString) \(arguments.joined(separator: " "))") } - // this shared global variable is safe because we're mutating it in a dispatch group - // https://developer.apple.com/documentation/foundation/process/1408746-terminationhandler - nonisolated(unsafe) var output = "" + let fd = dup(1) + let stdout = fdopen(fd, "rw") + defer { fclose(stdout) } + + // We need to use an unsafe transfer here to get the fd into our Sendable closure. + // This transfer is fine, because we guarantee that the code in the outputHandler + // is run before we continue the functions execution, where the fd is used again. + // See `process.waitUntilExit()` and the following `outputSync.wait()` + struct UnsafeTransfer: @unchecked Sendable { + let value: Value + } + + let outputMutex = Mutex("") let outputSync = DispatchGroup() - let outputQueue = DispatchQueue(label: "AWSLambdaPlugin.output") + let outputQueue = DispatchQueue(label: "AWSLambdaPackager.output") + let unsafeTransfer = UnsafeTransfer(value: stdout) let outputHandler = { @Sendable (data: Data?) in dispatchPrecondition(condition: .onQueue(outputQueue)) outputSync.enter() defer { outputSync.leave() } - guard let _output = data.flatMap({ String(decoding: $0, as: UTF8.self).trimmingCharacters(in: CharacterSet(["\n"])) }), !_output.isEmpty else { + guard + let _output = data.flatMap({ + String(data: $0, encoding: .utf8)?.trimmingCharacters(in: CharacterSet(["\n"])) + }), !_output.isEmpty + else { return } - output += _output + "\n" + outputMutex.withLock { output in + output += _output + "\n" + } switch logLevel { case .silent: @@ -60,24 +69,22 @@ struct Utils { case .debug(let outputIndent), .output(let outputIndent): print(String(repeating: " ", count: outputIndent), terminator: "") print(_output) - fflush(stdout) + fflush(unsafeTransfer.value) } } let pipe = Pipe() pipe.fileHandleForReading.readabilityHandler = { fileHandle in - outputQueue.async { - outputHandler(fileHandle.availableData) - } + outputQueue.async { outputHandler(fileHandle.availableData) } } let process = Process() process.standardOutput = pipe process.standardError = pipe - process.executableURL = executable + process.executableURL = URL(fileURLWithPath: executable.description) process.arguments = arguments - if let customWorkingDirectory { - process.currentDirectoryURL = customWorkingDirectory + if let workingDirectory = customWorkingDirectory { + process.currentDirectoryURL = URL(fileURLWithPath: workingDirectory.path()) } process.terminationHandler = { _ in outputQueue.async { @@ -91,24 +98,26 @@ struct Utils { // wait for output to be full processed outputSync.wait() + let output = outputMutex.withLock { $0 } + if process.terminationStatus != 0 { // print output on failure and if not already printed if logLevel < .output { print(output) fflush(stdout) } - throw ProcessError.processFailed([executable.absoluteString] + arguments, process.terminationStatus, output) + throw ProcessError.processFailed([executable.path()] + arguments, process.terminationStatus) } return output } enum ProcessError: Error, CustomStringConvertible { - case processFailed([String], Int32, String) + case processFailed([String], Int32) var description: String { switch self { - case .processFailed(let arguments, let code, _): + case .processFailed(let arguments, let code): return "\(arguments.joined(separator: " ")) failed with code \(code)" } } From aa9ccb47d164f6f47c7a76a0c0f8066bdf9ac642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Fri, 6 Sep 2024 14:16:14 +0200 Subject: [PATCH 06/10] add full foundation --- Plugins/AWSLambdaPackager/Plugin.swift | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index ed8efb0c..c4fb54a4 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -14,15 +14,7 @@ import PackagePlugin import Synchronization - -#if canImport(FoundationEssentials) -import FoundationEssentials -#else -import struct Foundation.URL -import class Foundation.ProcessInfo -import class Foundation.FileManager -import struct Foundation.ObjCBool -#endif +import Foundation @available(macOS 15.0, *) @main From 1254cace1ffb16dc0cfdea39dba00b2f389ef817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Fri, 6 Sep 2024 14:19:28 +0200 Subject: [PATCH 07/10] move macOS 15 guard to the `execute()` function only --- Plugins/AWSLambdaPackager/Plugin.swift | 2 -- Plugins/AWSLambdaPackager/PluginUtils.swift | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index c4fb54a4..647baee9 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -13,10 +13,8 @@ //===----------------------------------------------------------------------===// import PackagePlugin -import Synchronization import Foundation -@available(macOS 15.0, *) @main struct AWSLambdaPackager: CommandPlugin { func performCommand(context: PackagePlugin.PluginContext, arguments: [String]) async throws { diff --git a/Plugins/AWSLambdaPackager/PluginUtils.swift b/Plugins/AWSLambdaPackager/PluginUtils.swift index caa03326..5f6ef017 100644 --- a/Plugins/AWSLambdaPackager/PluginUtils.swift +++ b/Plugins/AWSLambdaPackager/PluginUtils.swift @@ -17,6 +17,7 @@ import PackagePlugin import Synchronization import Foundation +@available(macOS 15.0, *) struct Utils { @discardableResult static func execute( From d22b973a0f21d0908b7dfba26591af1bea68eeee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Fri, 6 Sep 2024 14:21:27 +0200 Subject: [PATCH 08/10] fix license file header --- Plugins/AWSLambdaPackager/PluginUtils.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Plugins/AWSLambdaPackager/PluginUtils.swift b/Plugins/AWSLambdaPackager/PluginUtils.swift index 5f6ef017..533a2372 100644 --- a/Plugins/AWSLambdaPackager/PluginUtils.swift +++ b/Plugins/AWSLambdaPackager/PluginUtils.swift @@ -1,4 +1,4 @@ -// ===----------------------------------------------------------------------===// +//===----------------------------------------------------------------------===// // // This source file is part of the SwiftAWSLambdaRuntime open source project // @@ -10,7 +10,7 @@ // // SPDX-License-Identifier: Apache-2.0 // -// ===----------------------------------------------------------------------===// +//===----------------------------------------------------------------------===// import Dispatch import PackagePlugin From d80fe9e481356c09eb3d2746ae644cd84aaf7656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Fri, 6 Sep 2024 14:24:38 +0200 Subject: [PATCH 09/10] clarify comment on usage of `UnsafeTransfer` --- Plugins/AWSLambdaPackager/PluginUtils.swift | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Plugins/AWSLambdaPackager/PluginUtils.swift b/Plugins/AWSLambdaPackager/PluginUtils.swift index 533a2372..6f60f7c4 100644 --- a/Plugins/AWSLambdaPackager/PluginUtils.swift +++ b/Plugins/AWSLambdaPackager/PluginUtils.swift @@ -35,9 +35,10 @@ struct Utils { defer { fclose(stdout) } // We need to use an unsafe transfer here to get the fd into our Sendable closure. - // This transfer is fine, because we guarantee that the code in the outputHandler - // is run before we continue the functions execution, where the fd is used again. - // See `process.waitUntilExit()` and the following `outputSync.wait()` + // This transfer is fine, because we write to the variable from a single SerialDispatchQueue here. + // We wait until the process is run below process.waitUntilExit(). + // This means no further writes to output will happen. + // This makes it save for us to read the output struct UnsafeTransfer: @unchecked Sendable { let value: Value } From d02a4604e2ed9c9727ffef06abdc8e6fc5cec1d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Stormacq?= Date: Fri, 6 Sep 2024 14:36:35 +0200 Subject: [PATCH 10/10] marking the plugin available only on macOS 15 --- Plugins/AWSLambdaPackager/Plugin.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index 647baee9..2e091c74 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -16,6 +16,7 @@ import PackagePlugin import Foundation @main +@available(macOS 15.0, *) struct AWSLambdaPackager: CommandPlugin { func performCommand(context: PackagePlugin.PluginContext, arguments: [String]) async throws { let configuration = try Configuration(context: context, arguments: arguments)