diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml index 4d6a5c2c5..1b29f629a 100644 --- a/.github/workflows/common.yml +++ b/.github/workflows/common.yml @@ -23,7 +23,7 @@ jobs: buildAndTest: name: Build and test the project if: github.repository == 'apple/container' - timeout-minutes: 75 + timeout-minutes: 90 runs-on: [self-hosted, macos, tahoe, ARM64] permissions: contents: read diff --git a/Makefile b/Makefile index 1f4248e9b..9ad62067d 100644 --- a/Makefile +++ b/Makefile @@ -133,6 +133,7 @@ $(STAGING_DIR): @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/container-core-images/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/resources)" + @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin)" @install "$(BUILD_BIN_DIR)/container" "$(join $(STAGING_DIR), bin/container)" @install "$(BUILD_BIN_DIR)/container-apiserver" "$(join $(STAGING_DIR), bin/container-apiserver)" @@ -146,6 +147,8 @@ $(STAGING_DIR): @install Sources/Plugins/MachineAPIServer/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/config.toml)" @install Sources/Plugins/MachineAPIServer/Resources/init "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/resources/init)" @install Sources/Plugins/MachineAPIServer/Resources/create-user.sh "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/resources/create-user.sh)" + @install "$(BUILD_BIN_DIR)/k8s" "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin/k8s)" + @install Sources/Plugins/K8s/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/k8s/config.toml)" @echo Install update script @install scripts/update-container.sh "$(join $(STAGING_DIR), bin/update-container.sh)" @@ -161,6 +164,7 @@ installer-pkg: $(STAGING_DIR) @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. --entitlements=signing/container-runtime-linux.entitlements "$(join $(STAGING_DIR), libexec/container/plugins/container-runtime-linux/bin/container-runtime-linux)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. --entitlements=signing/container-network-vmnet.entitlements "$(join $(STAGING_DIR), libexec/container/plugins/container-network-vmnet/bin/container-network-vmnet)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/bin/machine-apiserver)" + @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin/k8s)" @echo Creating application installer @pkgbuild --root "$(STAGING_DIR)" --identifier com.apple.container-installer --install-location /usr/local --version ${RELEASE_VERSION} $(PKG_PATH) diff --git a/Package.swift b/Package.swift index d46548cae..bb28a31f1 100644 --- a/Package.swift +++ b/Package.swift @@ -164,6 +164,34 @@ let package = Package( "ContainerResource", ] ), + .testTarget( + name: "K8sTests", + dependencies: [ + "k8s", + "ContainerResource", + "Yams", + ], + path: "Tests/K8sPluginTests" + ), + .executableTarget( + name: "k8s", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + .product(name: "Logging", package: "swift-log"), + .product(name: "Containerization", package: "containerization"), + .product(name: "ContainerizationOCI", package: "containerization"), + "ContainerAPIClient", + "ContainerLog", + "ContainerPersistence", + "ContainerResource", + "ContainerVersion", + "TerminalProgress", + "Yams", + ], + path: "Sources/Plugins/K8s", + exclude: ["config.toml"], + resources: [.process("Resources/kindnet.yaml")] + ), .executableTarget( name: "container-apiserver", dependencies: [ diff --git a/Sources/Plugins/K8s/K8sCommand.swift b/Sources/Plugins/K8s/K8sCommand.swift new file mode 100644 index 000000000..2076fa70d --- /dev/null +++ b/Sources/Plugins/K8s/K8sCommand.swift @@ -0,0 +1,59 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerVersion + +@main +struct K8sCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "k8s", + abstract: "Manage local Kubernetes development clusters (EXPERIMENTAL)", + discussion: """ + EXAMPLES: + Create a cluster by name and list clusters: + $ container k8s create --name my-cluster + $ container k8s list + + Switch between clusters: + $ container k8s create --name second-cluster + $ kubectl config use-context second-cluster + $ kubectl config use-context my-cluster + + Write the cluster context to an alternate configuration file: + $ container k8s write-config --name my-cluster --kubeconfig ~/.kube/my-cluster.kubeconfig + $ KUBECONFIG=~/.kube/my-cluster.kubeconfig kubectl cluster-info + + Load a local image into the cluster and run it: + $ container image pull docker.io/library/hello-world:latest + $ container image tag docker.io/library/hello-world:latest my-hello-world:latest + $ container k8s load-image --name my-cluster my-hello-world:latest + $ kubectl run hello-job --image=my-hello-world:latest --restart=Never --attach --rm -i + + Stop and delete the cluster: + $ container k8s delete --name my-cluster + """, + version: ReleaseVersion.singleLine(appName: "k8s"), + subcommands: [ + K8sCreate.self, + K8sDelete.self, + K8sList.self, + K8sLoadImage.self, + K8sStart.self, + K8sWriteConfig.self, + ] + ) +} diff --git a/Sources/Plugins/K8s/K8sCreate.swift b/Sources/Plugins/K8s/K8sCreate.swift new file mode 100644 index 000000000..cdce6cded --- /dev/null +++ b/Sources/Plugins/K8s/K8sCreate.swift @@ -0,0 +1,187 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerLog +import ContainerPersistence +import ContainerResource +import ContainerizationError +import Darwin +import Foundation +import Logging +import TerminalProgress + +struct K8sCreate: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "create", + abstract: "Create and start a local Kubernetes cluster" + ) + + @Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))") + var name: String = K8sHelper.defaultName + + @Flag(name: [.customLong("rm"), .long], help: "Remove the cluster container after it stops") + var remove: Bool = false + + @OptionGroup(title: "Resource options") + var resourceFlags: Flags.Resource + + @OptionGroup(title: "Registry options") + var registryFlags: Flags.Registry + + @OptionGroup(title: "Image fetch options") + var imageFetchFlags: Flags.ImageFetch + + @Option(help: "Node image reference (default: \(K8sHelper.nodeImage))") + var nodeImage: String = K8sHelper.nodeImage + + func run() async throws { + LoggingSystem.bootstrap { _ in StderrLogHandler() } + let log = Logger(label: K8sHelper.pluginName) + + guard ManagedContainer.nameValid(name) else { + throw ContainerizationError(.invalidArgument, message: "cluster name \(name) is not a valid container ID") + } + + let isTTY = isatty(FileHandle.standardError.fileDescriptor) == 1 + let progressConfig = try ProgressConfig( + showSpinner: isTTY, + showTasks: true, + showItems: true, + ignoreSmallSize: true, + totalTasks: 2, // fetch image, unpack image + clearOnFinish: isTTY, + outputMode: isTTY ? .ansi : .plain + ) + + let progress = ProgressBar(config: progressConfig) + defer { progress.finish() } + progress.start() + + let containerSystemConfig: ContainerSystemConfig = try await ConfigurationLoader.load() + try await K8sHelper.ensureImage(nodeImage: nodeImage, log: log, containerSystemConfig: containerSystemConfig) + + let fqdn = K8sHelper.fqdn(for: name, domain: containerSystemConfig.dns.domain) + let dns = Flags.DNS(domain: nil, nameservers: [], options: [], searchDomains: []) + + let management = Flags.Management( + arch: Arch.hostArchitecture().rawValue, + capAdd: ["ALL"], + capDrop: [], + cidfile: "", + detach: true, + dns: dns, + dnsDisabled: false, + entrypoint: nil, + initImage: nil, + kernel: nil, + kernelArgs: [], + labels: [ + "\(ResourceLabelKeys.plugin)=\(K8sHelper.pluginName)", + "\(ResourceLabelKeys.role)=\(K8sHelper.controlPlaneRoleName)", + ], + maskedPaths: [], + mounts: [], + name: name, + networks: [], + os: "linux", + platform: nil, + publishPorts: fqdn == nil ? [try await K8sHelper.clusterPort()] : [], + publishSockets: [], + readOnly: false, + readonlyPaths: [], + remove: remove, + rosetta: true, + runtime: nil, + ssh: false, + shmSize: nil, + tmpFs: [], + useInit: false, + virtualization: false, + volumes: [] + ) + + let updatedResource = K8sHelper.defaultedResourceFlags(resourceFlags) + let processFlags = Flags.Process(cwd: nil, env: K8sHelper.nodeProxyEnv(), envFile: [], gid: nil, interactive: false, tty: false, uid: nil, ulimits: [], user: nil) + + var (config, kernel, initfs) = try await Utility.containerConfigFromFlags( + id: name, + image: nodeImage, + arguments: [], + process: processFlags, + management: management, + resource: updatedResource, + registry: registryFlags, + imageFetch: imageFetchFlags, + containerSystemConfig: containerSystemConfig, + progressUpdate: progress.handler, + log: log + ) + + // Allow the node to modify /proc/sys (e.g. net.ipv4.ip_forward) during setup. + config.maskedPaths = [] + config.readonlyPaths = [] + + let client = ContainerClient() + let options = ContainerCreateOptions(autoRemove: remove) + try await client.create( + configuration: config, + options: options, + kernel: kernel, + initImage: initfs + ) + + progress.set(description: "Starting cluster") + let io = try ProcessIO.create(tty: false, interactive: false, detach: true) + defer { try? io.close() } + let process = try await client.bootstrap(id: name, stdio: io.stdio) + try await process.start() + try io.closeAfterStart() + + progress.set(description: "Waiting for node to boot") + try await K8sHelper.waitForNodeBooted(containerId: name, client: client, log: log) + + let snapshot = try await client.get(id: name) + guard let vmIP = snapshot.networks.first?.ipv4Address.address.description else { + throw ContainerizationError(.internalError, message: "no VM IP for control plane \(name)") + } + var sans = ["127.0.0.1"] + if let fqdn { sans.append(contentsOf: [vmIP, fqdn]) } + + progress.set(description: "Running kubeadm init") + try await K8sHelper.prepareNode(nodeID: name, client: client, log: log) + try await K8sHelper.bootstrapControlPlane( + nodeID: name, apiServerSANs: sans, advertiseAddress: vmIP, + client: client, log: log) + + progress.set(description: "Waiting for cluster to be ready") + try await K8sHelper.waitForReady(containerId: name, client: client, log: log) + + progress.set(description: "Writing kubeconfig") + do { + let rawConfig = try await K8sHelper.fetchConfig(containerId: name, client: client, log: log) + let kubeConfig = try await K8sHelper.transformConfig(rawConfig, containerId: name, fqdn: fqdn, client: client) + try K8sHelper.mergeConfig(kubeConfig, containerId: name, setCurrentContext: true, log: log) + } catch { + log.warning("failed to write kubeconfig", metadata: ["name": "\(name)", "error": "\(error)"]) + log.info("cluster is running; use 'container k8s write-config --name \(name)' to write the kubeconfig") + } + + progress.finish() + print(name) + } +} diff --git a/Sources/Plugins/K8s/K8sDelete.swift b/Sources/Plugins/K8s/K8sDelete.swift new file mode 100644 index 000000000..532a8044f --- /dev/null +++ b/Sources/Plugins/K8s/K8sDelete.swift @@ -0,0 +1,57 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerLog +import ContainerResource +import ContainerizationError +import Logging + +struct K8sDelete: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "delete", + abstract: "Delete a Kubernetes cluster", + aliases: ["rm"] + ) + + @Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))") + var name: String = K8sHelper.defaultName + + func run() async throws { + LoggingSystem.bootstrap { _ in StderrLogHandler() } + let log = Logger(label: K8sHelper.pluginName) + + let client = ContainerClient() + + if let container = try? await client.get(id: name) { + guard container.configuration.labels[ResourceLabelKeys.plugin] == K8sHelper.pluginName else { + log.error("container is not a k8s cluster, refusing delete", metadata: ["name": "\(name)"]) + throw ContainerizationError(.invalidArgument, message: "\(name) is not a k8s cluster") + } + } + + do { + try? await client.stop(id: name) + try await client.delete(id: name) + } catch let error as ContainerizationError where error.code == .notFound { + log.debug("cluster container not found, skipping delete", metadata: ["name": "\(name)"]) + } + + try K8sHelper.removeConfig(containerId: name, log: log) + print(name) + } +} diff --git a/Sources/Plugins/K8s/K8sHelper.swift b/Sources/Plugins/K8s/K8sHelper.swift new file mode 100644 index 000000000..314098400 --- /dev/null +++ b/Sources/Plugins/K8s/K8sHelper.swift @@ -0,0 +1,679 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerAPIClient +import ContainerPersistence +import ContainerResource +import ContainerVersion +import ContainerizationError +import ContainerizationOCI +import ContainerizationOS +import Darwin +import Foundation +import Logging +import SystemPackage +import Yams + +// MARK: - ListDisplayable + +protocol ListDisplayable { + static var tableHeader: [String] { get } + var tableRow: [String] { get } + var quietValue: String { get } +} + +// MARK: - TableOutput + +struct TableOutput: Sendable { + private let rows: [[String]] + private let spacing: Int + + init(rows: [[String]], spacing: Int = 2) { + self.rows = rows + self.spacing = spacing + } + + func format() -> String { + var output = "" + let maxLengths = self.maxLength() + + for rowIndex in 0.. [Int: Int] { + var output: [Int: Int] = [:] + for row in self.rows { + for (i, column) in row.enumerated() { + let currentMax = output[i] ?? 0 + output[i] = (column.count > currentMax) ? column.count : currentMax + } + } + return output + } +} + +// MARK: - K8sHelper + +struct K8sHelper { + static let pluginName: String = "k8s" + static let defaultName: String = "k8s-dev" + static let controlPlaneRoleName: String = "control-plane" + private static var defaultCPUs: Int64 { + Int64(max(ProcessInfo.processInfo.processorCount / 4, 2)) + } + + private static var defaultMemory: String { + let gb = Int(ProcessInfo.processInfo.physicalMemory / (1024 * 1024 * 1024)) / 4 + return "\(max(gb, 2))g" + } + + static let nodeImage = "docker.io/kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95" + private static let kubeconfigPath = "/etc/kubernetes/admin.conf" + private static let kubeconfigEnv = "KUBECONFIG=\(kubeconfigPath)" + private static let kubectlPath = "/bin/kubectl" + private static let kubeadmPath = "/usr/bin/kubeadm" + private static let ignorePreflightErrors = + "Swap,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables" + private static let podSubnet = "10.244.0.0/16" + // kubeadm default service subnet; must stay in sync if ClusterConfiguration.serviceSubnet is ever set. + private static let serviceSubnet = "10.96.0.0/12" + + // Proxy env var names forwarded from the host into the cluster container. + static let proxyEnvVars = ["HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"] + + // Returns proxy env vars with NO_PROXY augmented to bypass internal cluster CIDRs. + // Without this, kubelet routes apiserver traffic through the host proxy and times out. + static func nodeProxyEnv() -> [String] { + let bypassCIDRs = "192.168.0.0/16,\(podSubnet),\(serviceSubnet)" + let hostEnv = ProcessInfo.processInfo.environment + return proxyEnvVars.map { name in + guard name.uppercased() == "NO_PROXY" else { return name } + let existing = hostEnv[name] ?? hostEnv[name == "NO_PROXY" ? "no_proxy" : "NO_PROXY"] ?? "" + let augmented = existing.isEmpty ? bypassCIDRs : "\(existing),\(bypassCIDRs)" + return "\(name)=\(augmented)" + } + } + + private static let clusterContainerPort: UInt16 = 6443 + private static let clusterHostPortBase: UInt16 = 6445 + + private static func findAvailableHostPort(excluding: Set = []) throws -> UInt16 { + var port = clusterHostPortBase + while port < UInt16.max { + if excluding.contains(port) { + port += 1 + continue + } + let sock = Darwin.socket(AF_INET, SOCK_STREAM, 0) + guard sock >= 0 else { + throw ContainerizationError(.internalError, message: "socket() failed while probing for available port") + } + var addr = sockaddr_in() + addr.sin_family = sa_family_t(AF_INET) + addr.sin_port = port.bigEndian + addr.sin_addr.s_addr = INADDR_ANY + let available = withUnsafePointer(to: &addr) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { + Darwin.bind(sock, $0, socklen_t(MemoryLayout.size)) == 0 + } + } + Darwin.close(sock) + if available { return port } + port += 1 + } + throw ContainerizationError(.internalError, message: "no available host port found above \(clusterHostPortBase)") + } + + static func clusterPort() async throws -> String { + let snapshots = try await ContainerClient().list( + filters: ContainerListFilters(labels: [ResourceLabelKeys.plugin: pluginName]) + ) + let reserved = Set(snapshots.flatMap { $0.configuration.publishedPorts.map(\.hostPort) }) + let port = try findAvailableHostPort(excluding: reserved) + return "\(port):\(clusterContainerPort)" + } + + private static let kubeconfigDir: FilePath = FilePath( + FileManager.default.homeDirectoryForCurrentUser.path(percentEncoded: false) + ).appending(".kube") + + // MARK: - Resource defaults + + static func defaultedResourceFlags(_ flags: Flags.Resource) -> Flags.Resource { + var f = flags + if f.cpus == nil { f.cpus = defaultCPUs } + if f.memory == nil { f.memory = defaultMemory } + return f + } + + // MARK: - Image management + + static func ensureImage(nodeImage: String = K8sHelper.nodeImage, log: Logger, containerSystemConfig: ContainerSystemConfig) async throws { + do { + _ = try await ClientImage.get(reference: nodeImage, containerSystemConfig: containerSystemConfig) + log.debug("k8s node image present", metadata: ["ref": "\(nodeImage)"]) + return + } catch let error as ContainerizationError where error.code == .notFound { + log.info("Pulling k8s node image", metadata: ["ref": "\(nodeImage)"]) + } + let platform = try Platform(from: "linux/\(Arch.hostArchitecture().rawValue)") + _ = try await ClientImage.fetch( + reference: nodeImage, + platform: platform, + scheme: .auto, + containerSystemConfig: containerSystemConfig, + progressUpdate: nil) + } + + // MARK: - Node bootstrap + + private static func execCapture( + containerId: String, executable: String, arguments: [String], + client: ContainerClient + ) async throws -> (code: Int32, output: String) { + let pipe = Pipe() + let config = ProcessConfiguration( + executable: executable, arguments: arguments, environment: [], terminal: false) + let proc = try await client.createProcess( + containerId: containerId, processId: UUID().uuidString.lowercased(), + configuration: config, stdio: [nil, pipe.fileHandleForWriting, nil]) + try await proc.start() + pipe.fileHandleForWriting.closeFile() + let data = pipe.fileHandleForReading.readDataToEndOfFile() + try? pipe.fileHandleForReading.close() + let code = try await proc.wait() + return (code, String(data: data, encoding: .utf8) ?? "") + } + + static func prepareNode(nodeID: String, client: ContainerClient, log: Logger) async throws { + log.info("Preparing node", metadata: ["id": "\(nodeID)"]) + let result = try await execCapture( + containerId: nodeID, executable: "/bin/sh", + arguments: ["-c", nodePrepScript], client: client) + guard result.code == 0 else { + throw ContainerizationError(.internalError, message: "node prep failed on \(nodeID): \(result.output)") + } + } + + static func bootstrapControlPlane( + nodeID: String, apiServerSANs: [String], advertiseAddress: String, + client: ContainerClient, log: Logger + ) async throws { + let configYAML = initConfigYAML(advertiseAddress: advertiseAddress, certSANs: apiServerSANs) + var r = try await execCapture( + containerId: nodeID, executable: "/bin/sh", + arguments: ["-c", "cat > /etc/kubernetes/kubeadm-config.yaml <<'EOF'\n\(configYAML)\nEOF"], + client: client) + guard r.code == 0 else { + throw ContainerizationError(.internalError, message: "write kubeadm config failed on \(nodeID): \(r.output)") + } + + log.info("Running kubeadm init", metadata: ["node": "\(nodeID)"]) + r = try await execCapture( + containerId: nodeID, executable: kubeadmPath, + arguments: [ + "init", "--config", "/etc/kubernetes/kubeadm-config.yaml", + "--ignore-preflight-errors", ignorePreflightErrors, + ], + client: client) + guard r.code == 0 else { + throw ContainerizationError(.internalError, message: "kubeadm init failed on \(nodeID): \(r.output)") + } + + r = try await execCapture( + containerId: nodeID, executable: "/bin/sh", + arguments: ["-c", "mkdir -p /root/.kube && cp \(kubeconfigPath) /root/.kube/config"], + client: client) + guard r.code == 0 else { + throw ContainerizationError(.internalError, message: "failed to install root kubeconfig on \(nodeID): \(r.output)") + } + + log.info("Removing control-plane taint for single-node scheduling", metadata: ["node": "\(nodeID)"]) + _ = try await runProbe( + client: client, containerId: nodeID, + arguments: ["taint", "nodes", "--all", "node-role.kubernetes.io/control-plane-"]) + + log.info("Applying kindnet CNI", metadata: ["node": "\(nodeID)"]) + let manifest = try loadKindnetManifest() + let apply = + "cat > /tmp/kindnet.yaml <<'EOF'\n\(manifest)\nEOF\n" + + "\(kubeconfigEnv) kubectl apply -f /tmp/kindnet.yaml" + r = try await execCapture( + containerId: nodeID, executable: "/bin/sh", + arguments: ["-c", apply], client: client) + guard r.code == 0 else { + throw ContainerizationError(.internalError, message: "apply CNI failed on \(nodeID): \(r.output)") + } + } + + private static func loadKindnetManifest() throws -> String { + guard let url = Bundle.module.url(forResource: "kindnet", withExtension: "yaml"), + let contents = try? String(contentsOf: url, encoding: .utf8) + else { + throw ContainerizationError(.internalError, message: "kindnet manifest resource missing") + } + return contents + } + + private static let nodePrepScript: String = { + """ + set -e + mkdir -p /etc/containerd/conf.d + cat > /etc/containerd/conf.d/native-snapshotter.toml <<'EOF' + [plugins.'io.containerd.cri.v1.images'] + snapshotter = "native" + EOF + sysctl -w net.ipv4.ip_forward=1 2>/dev/null || true + sysctl -w net.bridge.bridge-nf-call-iptables=1 2>/dev/null || true + sysctl -w net.bridge.bridge-nf-call-ip6tables=1 2>/dev/null || true + systemctl restart containerd + ctr -n k8s.io images tag registry.k8s.io/pause:3.10 registry.k8s.io/pause:3.10.1 2>/dev/null || true + iptables -t mangle -A OUTPUT -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1220 + iptables -t mangle -A FORWARD -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1220 + """ + }() + + private static func initConfigYAML(advertiseAddress: String, certSANs: [String]) -> String { + let sans = certSANs.map { " - \($0)" }.joined(separator: "\n") + return """ + apiVersion: kubeadm.k8s.io/v1beta4 + kind: InitConfiguration + localAPIEndpoint: + advertiseAddress: \(advertiseAddress) + bindPort: 6443 + nodeRegistration: + criSocket: unix:///run/containerd/containerd.sock + --- + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration + kubernetesVersion: \(kubernetesVersion()) + networking: + podSubnet: \(podSubnet) + apiServer: + certSANs: + \(sans) + --- + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + cgroupDriver: systemd + failSwapOn: false + """ + } + + private static func kubernetesVersion() -> String { + let nameAndTag = nodeImage.split(separator: "@").first.map(String.init) ?? nodeImage + guard let ref = try? Reference.parse(nameAndTag), let tag = ref.tag else { return "v1.35" } + return tag + } + + // MARK: - FQDN detection + + static func fqdn(for name: String, domain: String?) -> String? { + if name.contains(".") { return name } + guard let domain, !domain.isEmpty else { return nil } + return "\(name).\(domain)" + } + + static func detectFQDN(name: String) async -> String? { + let domain = try? await ConfigurationLoader.load().dns.domain + return fqdn(for: name, domain: domain) + } + + // MARK: - Readiness + + static func waitForNodeBooted(containerId: String, client: ContainerClient, log: Logger) async throws { + let timeout = 120 + log.info("Waiting for node to boot", metadata: ["node": "\(containerId)"]) + for attempt in 1...timeout { + let result = try await execCapture( + containerId: containerId, executable: "/bin/sh", + arguments: ["-c", "test -S /run/containerd/containerd.sock"], client: client) + if result.code == 0 { return } + if attempt == timeout { + log.info("check container logs with 'container logs \(containerId)'") + throw ContainerizationError( + .timeout, + message: "node \(containerId) did not boot within \(timeout * 2)s: containerd socket not present at /run/containerd/containerd.sock" + ) + } + try await Task.sleep(for: .seconds(2)) + } + } + + private static func runProbe(client: ContainerClient, containerId: String, arguments: [String]) async throws -> Int32 { + let devNull = FileHandle(forWritingAtPath: "/dev/null") + defer { try? devNull?.close() } + let probe = ProcessConfiguration( + executable: kubectlPath, + arguments: arguments, + environment: [kubeconfigEnv], + terminal: false) + let proc = try await client.createProcess( + containerId: containerId, + processId: UUID().uuidString.lowercased(), + configuration: probe, + stdio: [nil, devNull, devNull]) + try await proc.start() + return try await proc.wait() + } + + static func waitForReady(containerId: String, client: ContainerClient, log: Logger) async throws { + let nodeReadyTimeout = 180 + let podReadyTimeout = 300 + + log.info("Waiting for control-plane node to become ready") + for attempt in 1...nodeReadyTimeout { + let code: Int32 + do { + code = try await runProbe( + client: client, containerId: containerId, + arguments: ["wait", "--for=condition=Ready", "node", "--all", "--timeout=2s"]) + } catch { + throw ContainerizationError( + .internalError, message: "k8s cluster \(containerId) stopped unexpectedly during startup: \(error)") + } + if code == 0 { break } + if attempt == nodeReadyTimeout { + log.info("inspect node state with 'container exec \(containerId) kubectl get nodes -o wide'") + throw ContainerizationError( + .timeout, + message: "k8s cluster \(containerId) control-plane node did not become Ready within \(nodeReadyTimeout * 2)s" + ) + } + try await Task.sleep(for: .seconds(2)) + } + + log.info("Waiting for kube-system pods to become ready") + for attempt in 1...podReadyTimeout { + let code: Int32 + do { + code = try await runProbe( + client: client, containerId: containerId, + arguments: ["wait", "--for=condition=Available", "deployment/coredns", "-n", "kube-system", "--timeout=2s"]) + } catch { + throw ContainerizationError( + .internalError, message: "k8s cluster \(containerId) stopped unexpectedly during startup: \(error)") + } + if code == 0 { return } + if attempt < podReadyTimeout { + try await Task.sleep(for: .seconds(2)) + } + } + log.info("inspect pod state with 'container exec \(containerId) kubectl get pods -n kube-system'") + throw ContainerizationError( + .timeout, + message: "k8s cluster \(containerId) kube-system pods did not become available within \(podReadyTimeout * 2)s" + ) + } + + // MARK: - Kubeconfig + + static func fetchConfig(containerId: String, client: ContainerClient, log: Logger) async throws -> KubeConfig { + log.info("Fetching kubeconfig", metadata: ["cluster": "\(containerId)"]) + + let container = try await client.get(id: containerId) + guard container.configuration.labels[ResourceLabelKeys.plugin] == pluginName else { + log.error("container is not a k8s cluster, refusing config fetch", metadata: ["name": "\(containerId)"]) + throw ContainerizationError(.invalidArgument, message: "\(containerId) is not a k8s cluster") + } + + let (exitCode, yaml) = try await execCapture( + containerId: containerId, executable: "/bin/cat", + arguments: [kubeconfigPath], client: client) + guard exitCode == 0 else { + throw ContainerizationError(.internalError, message: "failed to read kubeconfig from \(containerId): exit \(exitCode)") + } + do { + return try YAMLDecoder().decode(KubeConfig.self, from: yaml) + } catch { + throw ContainerizationError(.internalError, message: "failed to decode kubeconfig from \(containerId): \(error)") + } + } + + static func transformConfig(_ config: KubeConfig, containerId: String, fqdn: String?, client: ContainerClient) async throws -> KubeConfig { + var config = config + let serverAddress: String + if let fqdn { + serverAddress = "https://\(fqdn):6443" + } else { + let snapshot = try await client.get(id: containerId) + guard + let hostPort = snapshot.configuration.publishedPorts + .first(where: { $0.containerPort == clusterContainerPort })?.hostPort + else { + throw ContainerizationError(.internalError, message: "no published port for cluster \(containerId)") + } + serverAddress = "https://127.0.0.1:\(hostPort)" + } + for i in config.clusters.indices { + config.clusters[i].cluster.server = serverAddress + } + // Rename all entries to containerId. kubeadm uses fixed names ("kubernetes", + // "kubernetes-admin@kubernetes", etc.) rather than "default", so we rename + // unconditionally and fix up the cross-references in the context. + config.clusters = config.clusters.map { + var c = $0 + c.name = containerId + return c + } + config.users = config.users.map { + var u = $0 + u.name = containerId + return u + } + config.contexts = config.contexts.map { + var nc = $0 + nc.name = containerId + nc.context.cluster = containerId + nc.context.user = containerId + return nc + } + config.currentContext = containerId + return config + } + + static func resolveKubeconfigMergePath() -> FilePath { + let defaultPath = kubeconfigDir.appending("config") + guard let raw = Darwin.getenv("KUBECONFIG") else { return defaultPath } + let env = String(cString: raw) + guard !env.isEmpty else { return defaultPath } + let paths = env.split(separator: ":").map(String.init).filter { !$0.isEmpty } + guard !paths.isEmpty else { return defaultPath } + if paths.count == 1 { return FilePath(paths[0]) } + for p in paths where FileManager.default.fileExists(atPath: p) { + return FilePath(p) + } + return FilePath(paths[paths.count - 1]) + } + + static func mergeConfig(_ config: KubeConfig, containerId: String, targetPath: FilePath? = nil, setCurrentContext: Bool = false, log: Logger) throws { + let path = targetPath ?? resolveKubeconfigMergePath() + log.info("Writing kubeconfig", metadata: ["cluster": "\(containerId)", "path": "\(path)"]) + + let targetDir = path.removingLastComponent() + try FileManager.default.createDirectory(atPath: targetDir.string, withIntermediateDirectories: true) + + var existing: KubeConfig + if FileManager.default.fileExists(atPath: path.string) { + do { + let yaml = try String(contentsOfFile: path.string, encoding: .utf8) + existing = try YAMLDecoder().decode(KubeConfig.self, from: yaml) + } catch { + throw ContainerizationError( + .internalError, + message: "kubeconfig at \(path) could not be parsed: \(error)" + ) + } + } else { + existing = .empty + } + + existing.clusters.removeAll { $0.name == containerId } + existing.contexts.removeAll { $0.name == containerId } + existing.users.removeAll { $0.name == containerId } + + existing.clusters.append(contentsOf: config.clusters) + existing.contexts.append(contentsOf: config.contexts) + existing.users.append(contentsOf: config.users) + if setCurrentContext && existing.currentContext == nil { + existing.currentContext = containerId + } + + let output = try YAMLEncoder().encode(existing) + try output.write(toFile: path.string, atomically: true, encoding: .utf8) + } + + static func removeConfig(containerId: String, log: Logger) throws { + let path = resolveKubeconfigMergePath() + log.info("Removing kubeconfig", metadata: ["cluster": "\(containerId)", "path": "\(path)"]) + + guard FileManager.default.fileExists(atPath: path.string) else { return } + + let existing: KubeConfig + do { + let yaml = try String(contentsOfFile: path.string, encoding: .utf8) + existing = try YAMLDecoder().decode(KubeConfig.self, from: yaml) + } catch { + log.warning("kubeconfig exists but could not be parsed, skipping removal", metadata: ["path": "\(path)", "error": "\(error)"]) + return + } + + var updated = existing + + updated.clusters.removeAll { $0.name == containerId } + updated.contexts.removeAll { $0.name == containerId } + updated.users.removeAll { $0.name == containerId } + + if updated.currentContext == containerId { + updated.currentContext = nil + } + + let output = try YAMLEncoder().encode(updated) + try output.write(toFile: path.string, atomically: true, encoding: .utf8) + } + + // MARK: - List rows + + static func buildK8sRows(from snapshots: [ContainerSnapshot]) -> [K8sNodeResource] { + var controlPlanes: [ContainerSnapshot] = [] + var workers: [ContainerSnapshot] = [] + for snapshot in snapshots { + switch snapshot.configuration.labels[ResourceLabelKeys.role] { + case controlPlaneRoleName: controlPlanes.append(snapshot) + default: workers.append(snapshot) + } + } + + var rows: [K8sNodeResource] = [] + var assignedWorkerIDs = Set() + + for cp in controlPlanes.sorted(by: { $0.configuration.id < $1.configuration.id }) { + let clusterName = cp.configuration.id + rows.append(K8sNodeResource(clusterName: clusterName, snapshot: cp)) + let cpWorkers = + workers + .filter { $0.configuration.id.hasPrefix("\(clusterName)-worker-") } + .sorted { $0.configuration.id < $1.configuration.id } + for w in cpWorkers { + rows.append(K8sNodeResource(clusterName: clusterName, snapshot: w)) + assignedWorkerIDs.insert(w.configuration.id) + } + } + + for w + in workers + .filter({ !assignedWorkerIDs.contains($0.configuration.id) }) + .sorted(by: { $0.configuration.id < $1.configuration.id }) + { + let clusterName = w.configuration.id + .components(separatedBy: "-worker-").dropLast().joined(separator: "-worker-") + rows.append(K8sNodeResource(clusterName: clusterName, snapshot: w)) + } + + return rows + } + + static func renderTable(_ items: [T]) -> String { + var rows: [[String]] = [T.tableHeader] + for item in items { + rows.append(item.tableRow) + } + return TableOutput(rows: rows).format() + } + + // MARK: - Image reference helpers + + static func isShortName(_ reference: String) -> Bool { + guard let ref = try? Reference.parse(reference) else { return true } + return ref.domain == nil + } + + static func fqReference(_ reference: String) -> String { + guard let ref = try? Reference.parse(reference) else { return reference } + ref.normalize() + return ref.description + } +} + +// MARK: - K8sNodeResource + +struct K8sNodeResource: ManagedResource, ListDisplayable { + let clusterName: String + let snapshot: ContainerSnapshot + + // MARK: ManagedResource + var id: String { snapshot.configuration.id } + var name: String { snapshot.configuration.id } + var creationDate: Date { snapshot.configuration.creationDate } + var labels: ResourceLabels { (try? ResourceLabels(snapshot.configuration.labels)) ?? .init() } + + static func nameValid(_ name: String) -> Bool { ManagedContainer.nameValid(name) } + + static var tableHeader: [String] { + ["CLUSTER", "NODE", "ROLE", "STATE", "CPUS", "MEMORY", "ADDR", "PORTS"] + } + + var tableRow: [String] { + let role = snapshot.configuration.labels[ResourceLabelKeys.role] ?? "" + let addr = snapshot.networks.map { $0.ipv4Address.address.description }.joined(separator: ",") + let memoryMB = snapshot.configuration.resources.memoryInBytes / (1024 * 1024) + let ports = snapshot.configuration.publishedPorts + .map { "\($0.hostPort)->\($0.containerPort)" } + .joined(separator: ",") + return [ + clusterName, + snapshot.configuration.id, + role, + snapshot.status.rawValue, + "\(snapshot.configuration.resources.cpus)", + "\(memoryMB) MB", + addr, + ports, + ] + } + + var quietValue: String { snapshot.configuration.id } +} diff --git a/Sources/Plugins/K8s/K8sList.swift b/Sources/Plugins/K8s/K8sList.swift new file mode 100644 index 000000000..d24aec047 --- /dev/null +++ b/Sources/Plugins/K8s/K8sList.swift @@ -0,0 +1,39 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerLog +import ContainerResource +import Logging + +struct K8sList: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "list", + abstract: "List clusters and their nodes", + aliases: ["ls"] + ) + + func run() async throws { + LoggingSystem.bootstrap { _ in StderrLogHandler() } + + let snapshots = try await ContainerClient().list( + filters: ContainerListFilters(labels: [ResourceLabelKeys.plugin: K8sHelper.pluginName]) + ) + let rows = K8sHelper.buildK8sRows(from: snapshots) + print(K8sHelper.renderTable(rows)) + } +} diff --git a/Sources/Plugins/K8s/K8sLoadImage.swift b/Sources/Plugins/K8s/K8sLoadImage.swift new file mode 100644 index 000000000..8072f7026 --- /dev/null +++ b/Sources/Plugins/K8s/K8sLoadImage.swift @@ -0,0 +1,122 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerLog +import ContainerPersistence +import ContainerResource +import ContainerizationError +import ContainerizationOCI +import Foundation +import Logging +import SystemPackage + +struct K8sLoadImage: AsyncParsableCommand { + private static let ctrPath = "/usr/local/bin/ctr" + + static let configuration = CommandConfiguration( + commandName: "load-image", + abstract: "Load a container image into a cluster's containerd" + ) + + @Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))") + var name: String = K8sHelper.defaultName + + @Argument(help: "Image reference to load (e.g. demo-api:latest)") + var image: String + + @Option( + help: "Platform of the image to load (format: os/arch[/variant], default: linux/arm64)" + ) + var platform: String? + + func run() async throws { + LoggingSystem.bootstrap { _ in StderrLogHandler() } + let log = Logger(label: K8sHelper.pluginName) + + let tmpFile = FilePath(FileManager.default.temporaryDirectory.path(percentEncoded: false)) + .appending("k8s-image-\(UUID().uuidString).tar") + defer { try? FileManager.default.removeItem(atPath: tmpFile.string) } + + let containerSystemConfig: ContainerSystemConfig = try await ConfigurationLoader.load() + + let client = ContainerClient() + + // Guard: refuse to operate on a container not owned by this plugin. + let container = try await client.get(id: name) + guard container.configuration.labels[ResourceLabelKeys.plugin] == K8sHelper.pluginName else { + log.error("container is not a k8s cluster, refusing image load", metadata: ["name": "\(name)"]) + throw ContainerizationError(.invalidArgument, message: "\(name) is not a k8s cluster") + } + + log.info("Saving image", metadata: ["ref": "\(image)"]) + let fq = K8sHelper.fqReference(image) + let resolvedPlatform = try platform.map { try Platform(from: $0) } ?? Platform(from: "linux/\(Arch.hostArchitecture().rawValue)") + try await ClientImage.save(references: [fq], out: tmpFile.string, platform: resolvedPlatform, containerSystemConfig: containerSystemConfig) + + log.info("Importing image into cluster", metadata: ["target": "\(name)"]) + guard let inputHandle = FileHandle(forReadingAtPath: tmpFile.string) else { + throw ContainerizationError(.internalError, message: "failed to open image tar: \(tmpFile)") + } + defer { try? inputHandle.close() } + + let importConfig = ProcessConfiguration( + executable: Self.ctrPath, + arguments: ["--namespace", "k8s.io", "images", "import", "-"], + environment: [], + terminal: false + ) + let importProc = try await client.createProcess( + containerId: name, + processId: UUID().uuidString.lowercased(), + configuration: importConfig, + stdio: [inputHandle, nil, nil] + ) + try await importProc.start() + let importCode = try await importProc.wait() + guard importCode == 0 else { + throw ContainerizationError( + .internalError, + message: "ctr import exited \(importCode) on \(name)") + } + + // Tag with the fully-qualified docker.io/library/ name that kubelet expects, + // but only for short (unqualified) references. + if fq != image { + log.info("Tagging image for kubelet", metadata: ["short": "\(image)", "fq": "\(fq)"]) + let tagConfig = ProcessConfiguration( + executable: Self.ctrPath, + arguments: ["--namespace", "k8s.io", "images", "tag", fq, image], + environment: [], + terminal: false + ) + let tagProc = try await client.createProcess( + containerId: name, + processId: UUID().uuidString.lowercased(), + configuration: tagConfig, + stdio: [nil, nil, nil] + ) + try await tagProc.start() + let tagCode = try await tagProc.wait() + guard tagCode == 0 else { + throw ContainerizationError( + .internalError, + message: "ctr tag exited \(tagCode) on \(name)") + } + } + } +} diff --git a/Sources/Plugins/K8s/K8sStart.swift b/Sources/Plugins/K8s/K8sStart.swift new file mode 100644 index 000000000..adc98edf0 --- /dev/null +++ b/Sources/Plugins/K8s/K8sStart.swift @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerLog +import ContainerResource +import ContainerizationError +import Foundation +import Logging + +struct K8sStart: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "start", + abstract: "Start a stopped Kubernetes cluster" + ) + + @Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))") + var name: String = K8sHelper.defaultName + + func run() async throws { + LoggingSystem.bootstrap { _ in StderrLogHandler() } + let log = Logger(label: K8sHelper.pluginName) + + let client = ContainerClient() + let container = try await client.get(id: name) + + guard container.configuration.labels[ResourceLabelKeys.plugin] == K8sHelper.pluginName else { + log.error("container is not a k8s cluster, refusing start", metadata: ["name": "\(name)"]) + throw ContainerizationError(.invalidArgument, message: "\(name) is not a k8s cluster") + } + + if container.status == .running { + print(name) + return + } + + let io = try ProcessIO.create(tty: false, interactive: false, detach: true) + defer { try? io.close() } + let process = try await client.bootstrap(id: name, stdio: io.stdio) + try await process.start() + try io.closeAfterStart() + + try await K8sHelper.waitForNodeBooted(containerId: name, client: client, log: log) + try await K8sHelper.waitForReady(containerId: name, client: client, log: log) + + do { + let fqdn = await K8sHelper.detectFQDN(name: name) + let rawConfig = try await K8sHelper.fetchConfig(containerId: name, client: client, log: log) + let config = try await K8sHelper.transformConfig(rawConfig, containerId: name, fqdn: fqdn, client: client) + try K8sHelper.mergeConfig(config, containerId: name, log: log) + } catch { + log.warning("failed to write kubeconfig", metadata: ["name": "\(name)", "error": "\(error)"]) + log.info("cluster is running; use 'container k8s write-config --name \(name)' to write the kubeconfig") + } + + print(name) + } +} diff --git a/Sources/Plugins/K8s/K8sWriteConfig.swift b/Sources/Plugins/K8s/K8sWriteConfig.swift new file mode 100644 index 000000000..f7501febb --- /dev/null +++ b/Sources/Plugins/K8s/K8sWriteConfig.swift @@ -0,0 +1,47 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerLog +import Foundation +import Logging +import SystemPackage + +struct K8sWriteConfig: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "write-config", + abstract: "Write the cluster context to a Kubernetes configuration file" + ) + + @Option(name: .long, help: "Cluster name (default: \(K8sHelper.defaultName))") + var name: String = K8sHelper.defaultName + + @Option(name: .long, help: "Path to the kubeconfig file to write or append to (default: ~/.kube/config)") + var kubeconfig: String? + + func run() async throws { + LoggingSystem.bootstrap { _ in StderrLogHandler() } + let log = Logger(label: K8sHelper.pluginName) + + let targetPath = kubeconfig.map { FilePath($0) } + let client = ContainerClient() + let fqdn = await K8sHelper.detectFQDN(name: name) + let rawConfig = try await K8sHelper.fetchConfig(containerId: name, client: client, log: log) + let config = try await K8sHelper.transformConfig(rawConfig, containerId: name, fqdn: fqdn, client: client) + try K8sHelper.mergeConfig(config, containerId: name, targetPath: targetPath, log: log) + } +} diff --git a/Sources/Plugins/K8s/KubeConfig.swift b/Sources/Plugins/K8s/KubeConfig.swift new file mode 100644 index 000000000..755659857 --- /dev/null +++ b/Sources/Plugins/K8s/KubeConfig.swift @@ -0,0 +1,168 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +// MARK: - KubeConfig + +struct KubeConfig: Codable { + enum CodingKeys: String, CodingKey { + case apiVersion, kind, clusters, contexts, users + case currentContext = "current-context" + } + + var apiVersion: String = "v1" + var kind: String = "Config" + var clusters: [NamedCluster] = [] + var contexts: [NamedContext] = [] + var users: [NamedAuthInfo] = [] + var currentContext: String? + + init() {} + + init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + apiVersion = try c.decodeIfPresent(String.self, forKey: .apiVersion) ?? "v1" + kind = try c.decodeIfPresent(String.self, forKey: .kind) ?? "Config" + clusters = try c.decodeIfPresent([NamedCluster].self, forKey: .clusters) ?? [] + contexts = try c.decodeIfPresent([NamedContext].self, forKey: .contexts) ?? [] + users = try c.decodeIfPresent([NamedAuthInfo].self, forKey: .users) ?? [] + currentContext = try c.decodeIfPresent(String.self, forKey: .currentContext) + } + + static let empty = KubeConfig() +} + +// MARK: - NamedCluster + +struct NamedCluster: Codable { + var name: String + var cluster: Cluster +} + +// MARK: - Cluster + +struct Cluster: Codable { + enum CodingKeys: String, CodingKey { + case server + case tlsServerName = "tls-server-name" + case insecureSkipTLSVerify = "insecure-skip-tls-verify" + case certificateAuthority = "certificate-authority" + case certificateAuthorityData = "certificate-authority-data" + case proxyURL = "proxy-url" + } + + var server: String + var tlsServerName: String? + var insecureSkipTLSVerify: Bool? + var certificateAuthority: String? + var certificateAuthorityData: String? + var proxyURL: String? +} + +// MARK: - NamedContext + +struct NamedContext: Codable { + var name: String + var context: Context +} + +// MARK: - Context + +struct Context: Codable { + var cluster: String + var user: String + var namespace: String? +} + +// MARK: - NamedAuthInfo + +struct NamedAuthInfo: Codable { + enum CodingKeys: String, CodingKey { + case name + case authInfo = "user" + } + + var name: String + var authInfo: AuthInfo +} + +// MARK: - AuthInfo + +struct AuthInfo: Codable { + enum CodingKeys: String, CodingKey { + case clientCertificate = "client-certificate" + case clientCertificateData = "client-certificate-data" + case clientKey = "client-key" + case clientKeyData = "client-key-data" + case token + case tokenFile = "token-file" + case impersonate + case impersonateGroups = "impersonate-groups" + case impersonateUserExtra = "impersonate-user-extra" + case username + case password + case authProvider = "auth-provider" + case exec + } + + var clientCertificate: String? + var clientCertificateData: String? + var clientKey: String? + var clientKeyData: String? + var token: String? + var tokenFile: String? + var impersonate: String? + var impersonateGroups: [String]? + var impersonateUserExtra: [String: String]? + var username: String? + var password: String? + var authProvider: AuthProviderConfig? + var exec: ExecConfig? +} + +// MARK: - AuthProviderConfig + +struct AuthProviderConfig: Codable { + var name: String + var config: [String: String]? +} + +// MARK: - ExecConfig + +struct ExecConfig: Codable { + enum CodingKeys: String, CodingKey { + case command, args, env, apiVersion + case installHint = "installHint" + case provideClusterInfo = "provideClusterInfo" + case interactiveMode = "interactiveMode" + } + + var command: String + var args: [String]? + var env: [ExecEnvVar]? + var apiVersion: String + var installHint: String? + var provideClusterInfo: Bool? + var interactiveMode: String? +} + +// MARK: - ExecEnvVar + +struct ExecEnvVar: Codable { + var name: String + var value: String +} diff --git a/Sources/Plugins/K8s/Resources/kindnet.yaml b/Sources/Plugins/K8s/Resources/kindnet.yaml new file mode 100644 index 000000000..e6f0b31e6 --- /dev/null +++ b/Sources/Plugins/K8s/Resources/kindnet.yaml @@ -0,0 +1,135 @@ +# kindnetd networking manifest +# +# Pinned copy of kind's embedded default CNI manifest, transcribed verbatim +# from kubernetes-sigs/kind pkg/build/nodeimage/const_cni.go at tag v0.29.0: +# https://github.com/kubernetes-sigs/kind/blob/v0.29.0/pkg/build/nodeimage/const_cni.go +# +# The POD_SUBNET value must stay in sync with podSubnet in K8sHelper. +# +# kindnetd image pinned by the same kind release: docker.io/kindest/kindnetd:v20260528-9350166c +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kindnet +rules: + - apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - kindnet + - apiGroups: + - "" + resources: + - nodes + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kindnet +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kindnet +subjects: +- kind: ServiceAccount + name: kindnet + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kindnet + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kindnet + namespace: kube-system + labels: + tier: node + app: kindnet + k8s-app: kindnet +spec: + selector: + matchLabels: + app: kindnet + template: + metadata: + labels: + tier: node + app: kindnet + k8s-app: kindnet + spec: + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + serviceAccountName: kindnet + containers: + - name: kindnet-cni + image: docker.io/kindest/kindnetd:v20260528-9350166c + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_SUBNET + value: "10.244.0.0/16" + volumeMounts: + - name: cni-cfg + mountPath: /etc/cni/net.d + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: nri-plugin + mountPath: /var/run/nri + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_RAW", "NET_ADMIN"] + volumes: + - name: cni-cfg + hostPath: + path: /etc/cni/net.d + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: lib-modules + hostPath: + path: /lib/modules + - name: nri-plugin + hostPath: + path: /var/run/nri diff --git a/Sources/Plugins/K8s/config.toml b/Sources/Plugins/K8s/config.toml new file mode 100644 index 000000000..bd4b35e20 --- /dev/null +++ b/Sources/Plugins/K8s/config.toml @@ -0,0 +1,3 @@ +abstract = "Local Kubernetes development cluster management" +author = "Apple" +version = 0.1 diff --git a/Tests/IntegrationTests/K8s/TestK8sLoadImageSerial.swift b/Tests/IntegrationTests/K8s/TestK8sLoadImageSerial.swift new file mode 100644 index 000000000..59fd67234 --- /dev/null +++ b/Tests/IntegrationTests/K8s/TestK8sLoadImageSerial.swift @@ -0,0 +1,101 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerTestSupport +import Foundation +import Testing + +@Suite(.serialized) +struct TestK8sLoadImageSerial { + + private static let testImage = WarmupImage.alpine320.rawValue + + private func dumpNodeDiagnostics(_ f: ContainerFixture, node: String) { + print("=== NODE DIAGNOSTICS [\(node)] ===") + let cmds: [(label: String, args: [String])] = [ + ("ip-link", ["ip", "link", "show"]), + ("iptables-mss", ["iptables", "-t", "mangle", "-L", "-n", "-v"]), + ("containerd", ["systemctl", "status", "containerd", "--no-pager", "-l"]), + ("kubelet-log", ["journalctl", "-u", "kubelet", "--no-pager", "-n", "60"]), + ("crictl-images", ["crictl", "images"]), + ] + for (label, args) in cmds { + if let r = try? f.run(["exec", node] + args) { + let out = r.output.trimmingCharacters(in: .whitespacesAndNewlines) + let err = r.error.trimmingCharacters(in: .whitespacesAndNewlines) + print("[\(label)] exit=\(r.status)") + if !out.isEmpty { print(out) } + if !err.isEmpty { print("stderr: \(err)") } + } + } + print("=== END NODE DIAGNOSTICS ===") + } + + private func imageExistsInNode(_ f: ContainerFixture, node: String, image: String) throws -> Bool { + print("[k8s-load] ctr images list (node: \(node), checking: \(image))") + let result = try f.run(["exec", node, "ctr", "--namespace", "k8s.io", "images", "list"]) + print("[k8s-load] ctr images list exit=\(result.status) found=\(result.output.contains(image))") + guard result.status == 0 else { return false } + return result.output.contains(image) + } + + private func dumpEnv(_ f: ContainerFixture, clusterName: String) { + print("=== ENV DUMP [\(clusterName)] ===") + if let result = try? f.run(["system", "status"]) { + print("[system status]\n\(result.output)") + } + if let result = try? f.run(["list"]) { + print("[container list]\n\(result.output)") + } + if let result = try? f.run(["image", "list"]) { + print("[image list]\n\(result.output)") + } + if let result = try? f.run(["inspect", clusterName]) { + print("[inspect \(clusterName)]\nstdout: \(result.output)\nstderr: \(result.error)") + } + print("=== END ENV DUMP ===") + } + + @Test func testLoadImageIntoSingleNode() async throws { + try await ContainerFixture.with { f in + let name = "k8s-\(f.testID)" + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) } + + print("[k8s-load] k8s create --name \(name)") + let result = try f.run(["k8s", "create", "--name", name]) + print("[k8s-load] k8s create exit=\(result.status)") + if result.status != 0 { + print("[k8s-load] k8s create stderr: \(result.error)") + dumpEnv(f, clusterName: name) + dumpNodeDiagnostics(f, node: name) + } + try result.check() + + print("[k8s-load] pulling \(Self.testImage)") + try f.doPull(Self.testImage) + + print("[k8s-load] k8s load-image --name \(name) \(Self.testImage)") + let loadResult = try f.run(["k8s", "load-image", "--name", name, Self.testImage]) + print("[k8s-load] k8s load-image exit=\(loadResult.status)") + if loadResult.status != 0 { + print("[k8s-load] load-image stderr: \(loadResult.error)") + } + #expect(loadResult.status == 0) + + #expect(try imageExistsInNode(f, node: name, image: "alpine")) + } + } +} diff --git a/Tests/IntegrationTests/K8s/TestK8sNetworkingSerial.swift b/Tests/IntegrationTests/K8s/TestK8sNetworkingSerial.swift new file mode 100644 index 000000000..eeabaa6f6 --- /dev/null +++ b/Tests/IntegrationTests/K8s/TestK8sNetworkingSerial.swift @@ -0,0 +1,213 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerTestSupport +import Foundation +import Testing + +@Suite(.serialized) +struct TestK8sNetworkingSerial { + + private static let testImage = WarmupImage.alpine320.rawValue + + private func dumpNodeDiagnostics(_ f: ContainerFixture, node: String) { + print("=== NODE DIAGNOSTICS [\(node)] ===") + let cmds: [(label: String, args: [String])] = [ + ("ip-link", ["ip", "link", "show"]), + ("iptables-mss", ["iptables", "-t", "mangle", "-L", "-n", "-v"]), + ("containerd", ["systemctl", "status", "containerd", "--no-pager", "-l"]), + ("kubelet-log", ["journalctl", "-u", "kubelet", "--no-pager", "-n", "60"]), + ("crictl-images", ["crictl", "images"]), + ] + for (label, args) in cmds { + if let r = try? f.run(["exec", node] + args) { + let out = r.output.trimmingCharacters(in: .whitespacesAndNewlines) + let err = r.error.trimmingCharacters(in: .whitespacesAndNewlines) + print("[\(label)] exit=\(r.status)") + if !out.isEmpty { print(out) } + if !err.isEmpty { print("stderr: \(err)") } + } + } + print("=== END NODE DIAGNOSTICS ===") + } + + private func dumpEnv(_ f: ContainerFixture, clusterName: String) { + print("=== ENV DUMP [\(clusterName)] ===") + if let result = try? f.run(["system", "status"]) { + print("[system status]\n\(result.output)") + } + if let result = try? f.run(["list"]) { + print("[container list]\n\(result.output)") + } + if let result = try? f.run(["inspect", clusterName]) { + print("[inspect \(clusterName)] stdout: \(result.output) stderr: \(result.error)") + } + print("=== END ENV DUMP ===") + } + + @discardableResult + private func kubectl(_ f: ContainerFixture, node: String, args: [String]) throws -> (output: String, status: Int32) { + print("[k8s-net] kubectl \(args.joined(separator: " ")) (node: \(node))") + let result = try f.run(["exec", node, "kubectl"] + args) + print("[k8s-net] kubectl exit=\(result.status) output=\(result.output.prefix(120).trimmingCharacters(in: .whitespacesAndNewlines))") + let filteredStderr = result.error.components(separatedBy: "\n") + .filter { !$0.contains("Warning! Running debug build") && !$0.isEmpty } + .joined(separator: "\n") + if !filteredStderr.isEmpty { + print("[k8s-net] kubectl stderr: \(filteredStderr.prefix(300))") + } + return (result.output, result.status) + } + + private func waitForPod(_ f: ContainerFixture, node: String, podName: String, timeout: Int = 300) throws { + print("[k8s-net] waitForPod \(podName) on \(node) (timeout=\(timeout)s)") + let (_, status) = try kubectl( + f, node: node, + args: [ + "wait", "--for=condition=Ready", "pod/\(podName)", "--timeout=\(timeout)s", + ]) + guard status == 0 else { + let (podStatus, _) = try kubectl(f, node: node, args: ["get", "pod", podName, "--no-headers"]) + print("[k8s-net] pod \(podName) status: \(podStatus.trimmingCharacters(in: .whitespacesAndNewlines))") + let (podDesc, _) = try kubectl(f, node: node, args: ["describe", "pod", podName]) + print("[k8s-net] pod \(podName) describe:\n\(podDesc.prefix(1000))") + throw CommandError.executionFailed("pod \(podName) did not become ready within \(timeout)s") + } + print("[k8s-net] pod \(podName) is Ready") + } + + // Verify that a pod schedules, reaches Running, and can be exec'd into. + @Test func testPodsScheduleAndRun() async throws { + try await ContainerFixture.with { f in + let name = "k8s-\(f.testID)" + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) } + + print("[k8s-net] k8s create --name \(name)") + let result = try f.run(["k8s", "create", "--name", name]) + print("[k8s-net] k8s create exit=\(result.status)") + if result.status != 0 { + print("[k8s-net] k8s create stderr: \(result.error)") + dumpEnv(f, clusterName: name) + dumpNodeDiagnostics(f, node: name) + } + try result.check() + + print("[k8s-net] pulling \(Self.testImage)") + try f.doPull(Self.testImage) + + print("[k8s-net] k8s load-image --name \(name) \(Self.testImage)") + let loadResult = try f.run(["k8s", "load-image", "--name", name, Self.testImage]) + print("[k8s-net] k8s load-image exit=\(loadResult.status)") + if loadResult.status != 0 { print("[k8s-net] k8s load-image stderr: \(loadResult.error)") } + #expect(loadResult.status == 0) + + let (_, createStatus) = try kubectl( + f, node: name, + args: [ + "run", "test-pod", + "--image=\(Self.testImage)", + "--image-pull-policy=Never", + "--restart=Never", + "--", "sleep", "300", + ]) + #expect(createStatus == 0) + + try waitForPod(f, node: name, podName: "test-pod") + + let (output, execStatus) = try kubectl( + f, node: name, + args: [ + "exec", "test-pod", "--", "echo", "hello", + ]) + #expect(execStatus == 0) + #expect(output.contains("hello")) + } + } + + // Verify pod-to-service communication and CoreDNS resolution on a single node. + @Test func testPodToServiceCommunication() async throws { + try await ContainerFixture.with { f in + let name = "k8s-\(f.testID)" + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) } + + print("[k8s-net] k8s create --name \(name)") + let result = try f.run(["k8s", "create", "--name", name]) + print("[k8s-net] k8s create exit=\(result.status)") + if result.status != 0 { + print("[k8s-net] k8s create stderr: \(result.error)") + dumpEnv(f, clusterName: name) + } + try result.check() + + print("[k8s-net] pulling \(Self.testImage)") + try f.doPull(Self.testImage) + + print("[k8s-net] k8s load-image --name \(name) \(Self.testImage)") + let loadResult = try f.run(["k8s", "load-image", "--name", name, Self.testImage]) + print("[k8s-net] k8s load-image exit=\(loadResult.status)") + if loadResult.status != 0 { print("[k8s-net] k8s load-image stderr: \(loadResult.error)") } + #expect(loadResult.status == 0) + + // Server: alpine busybox httpd serving a static response. + let (_, serverStatus) = try kubectl( + f, node: name, + args: [ + "run", "server", + "--image=\(Self.testImage)", + "--image-pull-policy=Never", + "--restart=Never", + "--port=8080", + "--", "sh", "-c", + "while true; do printf 'HTTP/1.1 200 OK\\r\\nContent-Length: 2\\r\\nConnection: close\\r\\n\\r\\nok' | nc -l -p 8080; done", + ]) + #expect(serverStatus == 0) + try waitForPod(f, node: name, podName: "server") + + // Expose server as a ClusterIP service. + let (_, exposeStatus) = try kubectl( + f, node: name, + args: [ + "expose", "pod", "server", "--port=8080", "--name=server-svc", + ]) + #expect(exposeStatus == 0) + + // Client pod that stays alive so we can exec into it. + let (_, clientStatus) = try kubectl( + f, node: name, + args: [ + "run", "client", + "--image=\(Self.testImage)", + "--image-pull-policy=Never", + "--restart=Never", + "--", "sleep", "300", + ]) + #expect(clientStatus == 0) + try waitForPod(f, node: name, podName: "client") + + // Reach server via the service DNS name — exercises CoreDNS + kube-proxy. + print("[k8s-net] wget from client to server-svc:8080") + let (response, wgetStatus) = try kubectl( + f, node: name, + args: [ + "exec", "client", "--", "sh", "-c", + "sleep 2 && wget -qO- http://server-svc:8080", + ]) + print("[k8s-net] wget exit=\(wgetStatus) response=\(response.trimmingCharacters(in: .whitespacesAndNewlines))") + #expect(wgetStatus == 0) + #expect(response.contains("ok")) + } + } +} diff --git a/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift b/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift new file mode 100644 index 000000000..78dac3481 --- /dev/null +++ b/Tests/IntegrationTests/K8s/TestK8sRunSerial.swift @@ -0,0 +1,128 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerTestSupport +import Foundation +import SystemPackage +import Testing +import Yams + +@Suite(.serialized) +struct TestK8sRunSerial { + + private func dumpNodeDiagnostics(_ f: ContainerFixture, node: String) { + print("=== NODE DIAGNOSTICS [\(node)] ===") + let cmds: [(label: String, args: [String])] = [ + ("ip-link", ["ip", "link", "show"]), + ("iptables-mss", ["iptables", "-t", "mangle", "-L", "-n", "-v"]), + ("containerd", ["systemctl", "status", "containerd", "--no-pager", "-l"]), + ("kubelet-log", ["journalctl", "-u", "kubelet", "--no-pager", "-n", "60"]), + ("crictl-images", ["crictl", "images"]), + ] + for (label, args) in cmds { + if let r = try? f.run(["exec", node] + args) { + let out = r.output.trimmingCharacters(in: .whitespacesAndNewlines) + let err = r.error.trimmingCharacters(in: .whitespacesAndNewlines) + print("[\(label)] exit=\(r.status)") + if !out.isEmpty { print(out) } + if !err.isEmpty { print("stderr: \(err)") } + } + } + print("=== END NODE DIAGNOSTICS ===") + } + + private func loadKubeconfig() throws -> [String: Any] { + let path = FilePath(FileManager.default.homeDirectoryForCurrentUser.path) + .appending(".kube") + .appending("config") + let yaml = try String(contentsOfFile: path.string, encoding: .utf8) + guard let parsed = try Yams.load(yaml: yaml) as? [String: Any] else { + throw CommandError.executionFailed("could not parse kubeconfig at \(path.string)") + } + return parsed + } + + @Test func testRunSingleNode() async throws { + try await ContainerFixture.with { f in + let name = "k8s-\(f.testID)" + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) } + + print("[k8s-run] k8s create --name \(name)") + let result = try f.run(["k8s", "create", "--name", name]) + print("[k8s-run] k8s create exit=\(result.status)") + if result.status != 0 { + print("[k8s-run] k8s create stderr: \(result.error)") + dumpNodeDiagnostics(f, node: name) + } + try result.check() + #expect(result.output.contains(name)) + + let containerStatus = try f.getContainerStatus(name) + print("[k8s-run] container status=\(containerStatus)") + #expect(containerStatus == "running") + + let kubeconfig = try loadKubeconfig() + + let clusters = (kubeconfig["clusters"] as? [[String: Any]]) ?? [] + #expect(clusters.contains { $0["name"] as? String == name }) + + let contexts = (kubeconfig["contexts"] as? [[String: Any]]) ?? [] + #expect(contexts.contains { $0["name"] as? String == name }) + + let users = (kubeconfig["users"] as? [[String: Any]]) ?? [] + #expect(users.contains { $0["name"] as? String == name }) + } + } + + @Test func testConcurrentCreateGetsDifferentPorts() async throws { + try await ContainerFixture.with { f in + let name1 = "k8s-\(f.testID)-a" + let name2 = "k8s-\(f.testID)-b" + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name1]) } + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name2]) } + + print("[k8s-run] k8s create --name \(name1)") + let result1 = try f.run(["k8s", "create", "--name", name1]) + print("[k8s-run] k8s create exit=\(result1.status)") + if result1.status != 0 { + print("[k8s-run] k8s create stderr: \(result1.error)") + dumpNodeDiagnostics(f, node: name1) + } + #expect(result1.status == 0) + + print("[k8s-run] k8s create --name \(name2)") + let result2 = try f.run(["k8s", "create", "--name", name2]) + print("[k8s-run] k8s create exit=\(result2.status)") + if result2.status != 0 { + print("[k8s-run] k8s create stderr: \(result2.error)") + dumpNodeDiagnostics(f, node: name2) + } + #expect(result2.status == 0) + + #expect(try f.getContainerStatus(name1) == "running") + #expect(try f.getContainerStatus(name2) == "running") + + let port1 = try f.inspectContainer(name1).configuration.publishedPorts + .first(where: { $0.containerPort == 6443 })?.hostPort + let port2 = try f.inspectContainer(name2).configuration.publishedPorts + .first(where: { $0.containerPort == 6443 })?.hostPort + print("[k8s-run] port1=\(port1.map(String.init) ?? "nil") port2=\(port2.map(String.init) ?? "nil")") + #expect(port1 != nil) + #expect(port2 != nil) + #expect(port1 != port2) + } + } +} diff --git a/Tests/IntegrationTests/K8s/TestK8sWriteConfigSerial.swift b/Tests/IntegrationTests/K8s/TestK8sWriteConfigSerial.swift new file mode 100644 index 000000000..03fcba761 --- /dev/null +++ b/Tests/IntegrationTests/K8s/TestK8sWriteConfigSerial.swift @@ -0,0 +1,190 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerTestSupport +import Foundation +import SystemPackage +import Testing +import Yams + +@Suite(.serialized) +struct TestK8sWriteConfigSerial { + + private func dumpNodeDiagnostics(_ f: ContainerFixture, node: String) { + print("=== NODE DIAGNOSTICS [\(node)] ===") + let cmds: [(label: String, args: [String])] = [ + ("ip-link", ["ip", "link", "show"]), + ("iptables-mss", ["iptables", "-t", "mangle", "-L", "-n", "-v"]), + ("containerd", ["systemctl", "status", "containerd", "--no-pager", "-l"]), + ("kubelet-log", ["journalctl", "-u", "kubelet", "--no-pager", "-n", "60"]), + ("crictl-images", ["crictl", "images"]), + ] + for (label, args) in cmds { + if let r = try? f.run(["exec", node] + args) { + let out = r.output.trimmingCharacters(in: .whitespacesAndNewlines) + let err = r.error.trimmingCharacters(in: .whitespacesAndNewlines) + print("[\(label)] exit=\(r.status)") + if !out.isEmpty { print(out) } + if !err.isEmpty { print("stderr: \(err)") } + } + } + print("=== END NODE DIAGNOSTICS ===") + } + + private func kubeconfigPath() -> FilePath { + FilePath(FileManager.default.homeDirectoryForCurrentUser.path) + .appending(".kube") + .appending("config") + } + + private func loadKubeconfig() throws -> [String: Any] { + let yaml = try String(contentsOfFile: kubeconfigPath().string, encoding: .utf8) + guard let parsed = try Yams.load(yaml: yaml) as? [String: Any] else { + throw CommandError.executionFailed("could not parse kubeconfig") + } + return parsed + } + + private func dumpKubeconfig(for name: String) { + let path = kubeconfigPath() + guard FileManager.default.fileExists(atPath: path.string) else { + print("[k8s-cfg] \(path.lastComponent?.string ?? "config") does not exist") + return + } + guard let kubeconfig = try? loadKubeconfig() else { + print("[k8s-cfg] \(path.lastComponent?.string ?? "config") could not be parsed") + return + } + let clusters = (kubeconfig["clusters"] as? [[String: Any]])?.compactMap { $0["name"] as? String } ?? [] + let contexts = (kubeconfig["contexts"] as? [[String: Any]])?.compactMap { $0["name"] as? String } ?? [] + let current = kubeconfig["current-context"] as? String ?? "" + print("[k8s-cfg] kubeconfig: clusters=\(clusters) contexts=\(contexts) current-context=\(current)") + } + + private func dumpEnv(_ f: ContainerFixture, clusterName: String) { + print("=== ENV DUMP [\(clusterName)] ===") + if let result = try? f.run(["system", "status"]) { + print("[system status]\n\(result.output)") + } + if let result = try? f.run(["list"]) { + print("[container list]\n\(result.output)") + } + if let result = try? f.run(["inspect", clusterName]) { + print("[inspect \(clusterName)] stdout: \(result.output) stderr: \(result.error)") + } + print("=== END ENV DUMP ===") + } + + private func kubeconfigContains(name: String) -> Bool { + guard let kubeconfig = try? loadKubeconfig() else { return false } + let clusters = (kubeconfig["clusters"] as? [[String: Any]]) ?? [] + return clusters.contains { $0["name"] as? String == name } + } + + @Test func testWriteConfigMergesContext() async throws { + try await ContainerFixture.with { f in + let name = "k8s-\(f.testID)" + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) } + + print("[k8s-cfg] k8s create --name \(name)") + let result = try f.run(["k8s", "create", "--name", name]) + print("[k8s-cfg] k8s create exit=\(result.status)") + if result.status != 0 { + print("[k8s-cfg] k8s create stderr: \(result.error)") + dumpEnv(f, clusterName: name) + dumpNodeDiagnostics(f, node: name) + } + try result.check() + + print("[k8s-cfg] k8s write-config --name \(name)") + let writeResult = try f.run(["k8s", "write-config", "--name", name]) + print("[k8s-cfg] k8s write-config exit=\(writeResult.status)") + if writeResult.status != 0 { + print("[k8s-cfg] k8s write-config stderr: \(writeResult.error)") + dumpEnv(f, clusterName: name) + } + #expect(writeResult.status == 0) + + let kubeconfig = try loadKubeconfig() + + let clusters = (kubeconfig["clusters"] as? [[String: Any]]) ?? [] + #expect(clusters.contains { $0["name"] as? String == name }) + + let contexts = (kubeconfig["contexts"] as? [[String: Any]]) ?? [] + #expect(contexts.contains { $0["name"] as? String == name }) + + let users = (kubeconfig["users"] as? [[String: Any]]) ?? [] + #expect(users.contains { $0["name"] as? String == name }) + } + } + + @Test func testWriteConfigIsIdempotent() async throws { + try await ContainerFixture.with { f in + let name = "k8s-\(f.testID)" + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) } + + print("[k8s-cfg] k8s create --name \(name)") + let result = try f.run(["k8s", "create", "--name", name]) + print("[k8s-cfg] k8s create exit=\(result.status)") + if result.status != 0 { + print("[k8s-cfg] k8s create stderr: \(result.error)") + dumpEnv(f, clusterName: name) + dumpNodeDiagnostics(f, node: name) + } + try result.check() + + print("[k8s-cfg] k8s write-config --name \(name) (first)") + _ = try f.run(["k8s", "write-config", "--name", name]) + print("[k8s-cfg] k8s write-config --name \(name) (second)") + let secondResult = try f.run(["k8s", "write-config", "--name", name]) + print("[k8s-cfg] k8s write-config (second) exit=\(secondResult.status)") + #expect(secondResult.status == 0) + + let kubeconfig = try loadKubeconfig() + let clusters = (kubeconfig["clusters"] as? [[String: Any]]) ?? [] + let matchingClusters = clusters.filter { $0["name"] as? String == name } + #expect(matchingClusters.count == 1) + } + } + + @Test func testDeleteClusterCleansUpKubeconfig() async throws { + try await ContainerFixture.with { f in + let name = "k8s-\(f.testID)" + f.addCleanup { _ = try? f.run(["k8s", "delete", "--name", name]) } + + print("[k8s-cfg] k8s create --name \(name)") + let result = try f.run(["k8s", "create", "--name", name]) + print("[k8s-cfg] k8s create exit=\(result.status)") + if result.status != 0 { + print("[k8s-cfg] k8s create stderr: \(result.error)") + dumpEnv(f, clusterName: name) + dumpNodeDiagnostics(f, node: name) + } + try result.check() + + print("[k8s-cfg] k8s write-config --name \(name)") + _ = try f.run(["k8s", "write-config", "--name", name]) + #expect(kubeconfigContains(name: name)) + + print("[k8s-cfg] k8s delete --name \(name)") + let deleteResult = try f.run(["k8s", "delete", "--name", name]) + print("[k8s-cfg] k8s delete exit=\(deleteResult.status) stderr=\(deleteResult.error)") + #expect(deleteResult.status == 0) + + #expect(!kubeconfigContains(name: name)) + } + } +} diff --git a/Tests/K8sPluginTests/K8sListTests.swift b/Tests/K8sPluginTests/K8sListTests.swift new file mode 100644 index 000000000..44b87a8a1 --- /dev/null +++ b/Tests/K8sPluginTests/K8sListTests.swift @@ -0,0 +1,277 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerResource +import Foundation +import Testing +import Yams + +@testable import k8s + +// MARK: - Fixtures + +private let fixtureDefaultCPUs: Int = max(ProcessInfo.processInfo.processorCount / 4, 2) +private let fixtureDefaultMemoryMiB: UInt64 = { + let gb = max(Int(ProcessInfo.processInfo.physicalMemory / (1024 * 1024 * 1024)) / 4, 2) + return UInt64(gb * 1024) +}() + +private func makeSnapshot( + id: String, + role: String, + status: RuntimeStatus = .running, + cpus: Int = 2, + memoryMiB: UInt64 = 2048, + addr: String = "" +) throws -> ContainerSnapshot { + let labelsJSON = #"{"com.apple.container.plugin":"k8s","com.apple.container.resource.role":"\#(role)"}"# + let networksJSON: String + if addr.isEmpty { + networksJSON = "[]" + } else { + networksJSON = #"[{"network":"bridge","hostname":"\#(id)","ipv4Address":"\#(addr)/24","ipv4Gateway":"10.0.0.254"}]"# + } + let json = """ + { + "configuration": { + "id": "\(id)", + "image": { + "reference": "docker.io/kindest/node:v1.35.5", + "descriptor": {"mediaType":"","digest":"sha256:abc","size":0} + }, + "initProcess": {"executable":"/bin/sh","arguments":[],"environment":[],"workingDirectory":"/","terminal":false,"user":{"id":{"uid":0,"gid":0}},"supplementalGroups":[],"rlimits":[]}, + "resources": {"cpus":\(cpus),"memoryInBytes":\(memoryMiB * 1024 * 1024)}, + "labels": \(labelsJSON) + }, + "status": "\(status.rawValue)", + "networks": \(networksJSON) + } + """ + return try JSONDecoder().decode(ContainerSnapshot.self, from: Data(json.utf8)) +} + +private func makeControlPlane(_ name: String, cpus: Int = fixtureDefaultCPUs, memoryMiB: UInt64 = fixtureDefaultMemoryMiB, addr: String = "") throws -> ContainerSnapshot { + try makeSnapshot(id: name, role: K8sHelper.controlPlaneRoleName, cpus: cpus, memoryMiB: memoryMiB, addr: addr) +} + +private func makeWorker(_ id: String, cpus: Int = max(fixtureDefaultCPUs / 2, 1), memoryMiB: UInt64 = max(fixtureDefaultMemoryMiB / 2, 512)) throws -> ContainerSnapshot { + try makeSnapshot(id: id, role: "worker", cpus: cpus, memoryMiB: memoryMiB) +} + +// MARK: - K8sNodeResource header / row shape + +@Suite("K8sNodeResource") +struct K8sNodeRowTests { + @Test func tableHeaderHasEightColumns() throws { + #expect(K8sNodeResource.tableHeader.count == 8) + } + + @Test func tableHeaderLabels() throws { + #expect(K8sNodeResource.tableHeader == ["CLUSTER", "NODE", "ROLE", "STATE", "CPUS", "MEMORY", "ADDR", "PORTS"]) + } + + @Test func rowColumnCountMatchesHeader() throws { + let snapshot = try makeControlPlane("dev") + let row = K8sNodeResource(clusterName: "dev", snapshot: snapshot) + #expect(row.tableRow.count == K8sNodeResource.tableHeader.count) + } + + @Test func controlPlaneRowValues() throws { + let cpus = fixtureDefaultCPUs + let memoryMiB = fixtureDefaultMemoryMiB + let snapshot = try makeControlPlane("dev", cpus: cpus, memoryMiB: memoryMiB, addr: "10.0.0.1") + let row = K8sNodeResource(clusterName: "dev", snapshot: snapshot) + let cols = row.tableRow + #expect(cols[0] == "dev") // CLUSTER + #expect(cols[1] == "dev") // NODE + #expect(cols[2] == K8sHelper.controlPlaneRoleName) // ROLE + #expect(cols[3] == "running") // STATE + #expect(cols[4] == "\(cpus)") // CPUS + #expect(cols[5] == "\(memoryMiB) MB") // MEMORY + #expect(cols[6].contains("10.0.0.1")) // ADDR + // cols[7] PORTS: no publishedPorts in fixture → empty string + #expect(cols[7] == "") + } + + @Test func workerRowValues() throws { + let cpus = max(fixtureDefaultCPUs / 2, 1) + let memoryMiB = max(fixtureDefaultMemoryMiB / 2, 512) + let snapshot = try makeWorker("dev-worker-1", cpus: cpus, memoryMiB: memoryMiB) + let row = K8sNodeResource(clusterName: "dev", snapshot: snapshot) + let cols = row.tableRow + #expect(cols[0] == "dev") // CLUSTER + #expect(cols[1] == "dev-worker-1") // NODE + #expect(cols[2] == "worker") // ROLE + #expect(cols[4] == "\(cpus)") // CPUS + #expect(cols[5] == "\(memoryMiB) MB") // MEMORY + } + + @Test func quietValueIsNodeID() throws { + let snapshot = try makeWorker("dev-worker-2") + let row = K8sNodeResource(clusterName: "dev", snapshot: snapshot) + #expect(row.quietValue == "dev-worker-2") + } +} + +// MARK: - buildK8sRows ordering and grouping + +@Suite("K8sHelper.buildK8sRows") +struct BuildK8sRowsTests { + @Test func emptyInputProducesNoRows() { + #expect(K8sHelper.buildK8sRows(from: []).isEmpty) + } + + @Test func singleControlPlaneAlone() throws { + let cp = try makeControlPlane("dev") + let rows = K8sHelper.buildK8sRows(from: [cp]) + #expect(rows.count == 1) + #expect(rows[0].snapshot.configuration.id == "dev") + } + + @Test func controlPlaneFollowedByItsWorkers() throws { + let cp = try makeControlPlane("dev") + let w1 = try makeWorker("dev-worker-1") + let w2 = try makeWorker("dev-worker-2") + let rows = K8sHelper.buildK8sRows(from: [w2, w1, cp]) + #expect(rows.count == 3) + #expect(rows[0].snapshot.id == "dev") + #expect(rows[1].snapshot.id == "dev-worker-1") + #expect(rows[2].snapshot.id == "dev-worker-2") + } + + @Test func workersGroupedUnderCorrectControlPlane() throws { + let cp1 = try makeControlPlane("alpha") + let cp2 = try makeControlPlane("beta") + let w1 = try makeWorker("alpha-worker-1") + let w2 = try makeWorker("beta-worker-1") + let rows = K8sHelper.buildK8sRows(from: [w2, cp2, w1, cp1]) + #expect(rows[0].snapshot.id == "alpha") + #expect(rows[1].snapshot.id == "alpha-worker-1") + #expect(rows[1].clusterName == "alpha") + #expect(rows[2].snapshot.id == "beta") + #expect(rows[3].snapshot.id == "beta-worker-1") + #expect(rows[3].clusterName == "beta") + } + + @Test func multipleClustersAreSortedByName() throws { + let cpZ = try makeControlPlane("zebra") + let cpA = try makeControlPlane("apple") + let rows = K8sHelper.buildK8sRows(from: [cpZ, cpA]) + #expect(rows[0].snapshot.id == "apple") + #expect(rows[1].snapshot.id == "zebra") + } + + @Test func orphanedWorkerAppearsAtEnd() throws { + let cp = try makeControlPlane("dev") + let orphan = try makeWorker("old-worker-1") + let rows = K8sHelper.buildK8sRows(from: [orphan, cp]) + #expect(rows.count == 2) + #expect(rows[0].snapshot.id == "dev") + #expect(rows[1].snapshot.id == "old-worker-1") + #expect(rows[1].clusterName == "old") + } + + @Test func orphanedWorkerClusterNameDerivedFromID() throws { + let orphan = try makeWorker("mycluster-worker-3") + let rows = K8sHelper.buildK8sRows(from: [orphan]) + #expect(rows.count == 1) + #expect(rows[0].clusterName == "mycluster") + } + + @Test func clusterNameWithWorkerInItIsHandledCorrectly() throws { + let cp = try makeControlPlane("foo-worker") + let w = try makeWorker("foo-worker-worker-1") + let rows = K8sHelper.buildK8sRows(from: [w, cp]) + #expect(rows.count == 2) + #expect(rows[0].snapshot.id == "foo-worker") + #expect(rows[1].snapshot.id == "foo-worker-worker-1") + #expect(rows[1].clusterName == "foo-worker") + } + + @Test func workerNotAssignedToWrongCluster() throws { + let cp = try makeControlPlane("dev") + let w = try makeWorker("dev2-worker-1") + let rows = K8sHelper.buildK8sRows(from: [w, cp]) + #expect(rows[0].snapshot.id == "dev") + #expect(rows[1].snapshot.id == "dev2-worker-1") + #expect(rows[1].clusterName == "dev2") + } +} + +// MARK: - K8sHelper.fqdn + +@Suite("K8sHelper.fqdn") +struct FQDNTests { + @Test func nilDomainReturnsNil() { + #expect(K8sHelper.fqdn(for: "dev", domain: nil) == nil) + } + + @Test func emptyDomainReturnsNil() { + #expect(K8sHelper.fqdn(for: "dev", domain: "") == nil) + } + + @Test func nameWithDotReturnedAsIs() { + #expect(K8sHelper.fqdn(for: "dev.local", domain: "example.com") == "dev.local") + } + + @Test func plainNameGetsDomainAppended() { + #expect(K8sHelper.fqdn(for: "dev", domain: "example.com") == "dev.example.com") + } +} + +// MARK: - KubeConfig + +@Suite("KubeConfig") +struct KubeKubeConfigTests { + private func sampleYAML(clusterName: String = "default") -> String { + """ + apiVersion: v1 + kind: Config + clusters: + - name: \(clusterName) + cluster: + server: https://127.0.0.1:6443 + certificate-authority-data: dGVzdA== + contexts: + - name: \(clusterName) + context: + cluster: \(clusterName) + user: \(clusterName) + current-context: \(clusterName) + users: + - name: \(clusterName) + user: + client-certificate-data: dGVzdA== + client-key-data: dGVzdA== + """ + } + + @Test func decodeRoundTrip() throws { + let config = try YAMLDecoder().decode(KubeConfig.self, from: sampleYAML()) + #expect(config.apiVersion == "v1") + #expect(config.kind == "Config") + #expect(config.clusters.count == 1) + #expect(config.clusters[0].cluster.server == "https://127.0.0.1:6443") + #expect(config.contexts.count == 1) + #expect(config.users.count == 1) + #expect(config.currentContext == "default") + } + + @Test func emptyConfigEncodesWithoutError() throws { + let output = try YAMLEncoder().encode(KubeConfig.empty) + #expect(!output.isEmpty) + } +} diff --git a/Tests/K8sPluginTests/KubeconfigMergeTests.swift b/Tests/K8sPluginTests/KubeconfigMergeTests.swift new file mode 100644 index 000000000..58e91c80d --- /dev/null +++ b/Tests/K8sPluginTests/KubeconfigMergeTests.swift @@ -0,0 +1,494 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Darwin +import Foundation +import Logging +import SystemPackage +import Testing +import Yams + +@testable import k8s + +// MARK: - Helpers + +private func makeTempFile() throws -> (FilePath, cleanup: () -> Void) { + let path = FilePath(FileManager.default.temporaryDirectory.path) + .appending("kubeconfig-test-\(UUID().uuidString)") + return (path, { try? FileManager.default.removeItem(atPath: path.string) }) +} + +private func decode(_ yaml: String) throws -> KubeConfig { + try YAMLDecoder().decode(KubeConfig.self, from: yaml) +} + +private func encode(_ config: KubeConfig) throws -> String { + try YAMLEncoder().encode(config) +} + +private let log = Logger(label: "test") + +// MARK: - Round-trip fidelity + +@Suite("KubeConfig round-trip fidelity") +struct KubeconfigRoundTripTests { + + @Test func execAuthRoundTrip() throws { + let yaml = """ + apiVersion: v1 + kind: Config + clusters: + - name: gke-cluster + cluster: + server: https://1.2.3.4 + certificate-authority-data: dGVzdA== + contexts: + - name: gke-context + context: + cluster: gke-cluster + user: gke-user + namespace: production + current-context: gke-context + users: + - name: gke-user + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + command: gke-gcloud-auth-plugin + args: + - --some-flag + env: + - name: USE_GKE_GCLOUD_AUTH_PLUGIN + value: "True" + installHint: Install gke-gcloud-auth-plugin + provideClusterInfo: true + interactiveMode: IfAvailable + """ + let config = try decode(yaml) + let user = try #require(config.users.first) + let exec = try #require(user.authInfo.exec) + #expect(exec.command == "gke-gcloud-auth-plugin") + #expect(exec.args == ["--some-flag"]) + #expect(exec.env?.first?.name == "USE_GKE_GCLOUD_AUTH_PLUGIN") + #expect(exec.env?.first?.value == "True") + #expect(exec.installHint == "Install gke-gcloud-auth-plugin") + #expect(exec.provideClusterInfo == true) + #expect(exec.interactiveMode == "IfAvailable") + + // Re-encode and decode again — fields must survive the round-trip + let reencoded = try encode(config) + let redecoded = try decode(reencoded) + let exec2 = try #require(redecoded.users.first?.authInfo.exec) + #expect(exec2.command == exec.command) + #expect(exec2.args == exec.args) + #expect(exec2.env?.first?.name == exec.env?.first?.name) + #expect(exec2.provideClusterInfo == exec.provideClusterInfo) + #expect(redecoded.contexts.first?.context.namespace == "production") + } + + @Test func tokenAuthRoundTrip() throws { + let yaml = """ + apiVersion: v1 + kind: Config + clusters: + - name: my-cluster + cluster: + server: https://1.2.3.4 + contexts: + - name: my-context + context: + cluster: my-cluster + user: my-user + current-context: my-context + users: + - name: my-user + user: + token: supersecrettoken + """ + let config = try decode(yaml) + #expect(config.users.first?.authInfo.token == "supersecrettoken") + + let redecoded = try decode(try encode(config)) + #expect(redecoded.users.first?.authInfo.token == "supersecrettoken") + } + + @Test func authProviderRoundTrip() throws { + let yaml = """ + apiVersion: v1 + kind: Config + clusters: + - name: legacy-gke + cluster: + server: https://1.2.3.4 + contexts: + - name: legacy-context + context: + cluster: legacy-gke + user: legacy-user + current-context: legacy-context + users: + - name: legacy-user + user: + auth-provider: + name: gcp + config: + cmd-path: /usr/lib/google-cloud-sdk/bin/gcloud + token-key: '{.credential.access_token}' + """ + let config = try decode(yaml) + let provider = try #require(config.users.first?.authInfo.authProvider) + #expect(provider.name == "gcp") + #expect(provider.config?["cmd-path"] == "/usr/lib/google-cloud-sdk/bin/gcloud") + + let redecoded = try decode(try encode(config)) + let provider2 = try #require(redecoded.users.first?.authInfo.authProvider) + #expect(provider2.name == provider.name) + #expect(provider2.config?["cmd-path"] == provider.config?["cmd-path"]) + } + + @Test func clusterFieldsRoundTrip() throws { + let yaml = """ + apiVersion: v1 + kind: Config + clusters: + - name: proxied-cluster + cluster: + server: https://1.2.3.4 + tls-server-name: my-server.example.com + insecure-skip-tls-verify: true + proxy-url: http://proxy.example.com:8080 + contexts: + - name: proxied-context + context: + cluster: proxied-cluster + user: proxied-user + current-context: proxied-context + users: + - name: proxied-user + user: + token: abc + """ + let config = try decode(yaml) + let cluster = try #require(config.clusters.first?.cluster) + #expect(cluster.tlsServerName == "my-server.example.com") + #expect(cluster.insecureSkipTLSVerify == true) + #expect(cluster.proxyURL == "http://proxy.example.com:8080") + + let redecoded = try decode(try encode(config)) + let cluster2 = try #require(redecoded.clusters.first?.cluster) + #expect(cluster2.tlsServerName == cluster.tlsServerName) + #expect(cluster2.insecureSkipTLSVerify == cluster.insecureSkipTLSVerify) + #expect(cluster2.proxyURL == cluster.proxyURL) + } +} + +// MARK: - mergeConfig behavior + +@Suite("K8sHelper.mergeConfig") +struct MergeConfigTests { + + private func makeConfig(clusterName: String, server: String = "https://127.0.0.1:6443") -> KubeConfig { + var config = KubeConfig() + config.clusters = [NamedCluster(name: clusterName, cluster: Cluster(server: server, certificateAuthorityData: "dGVzdA=="))] + config.contexts = [NamedContext(name: clusterName, context: Context(cluster: clusterName, user: clusterName))] + config.users = [NamedAuthInfo(name: clusterName, authInfo: AuthInfo(clientCertificateData: "dGVzdA==", clientKeyData: "dGVzdA=="))] + config.currentContext = clusterName + return config + } + + @Test func mergeIntoEmptyFileCreatesFile() throws { + let (path, cleanup) = try makeTempFile() + defer { cleanup() } + + try K8sHelper.mergeConfig(makeConfig(clusterName: "dev"), containerId: "dev", targetPath: path, setCurrentContext: true, log: log) + + let written = try decode(String(contentsOfFile: path.string, encoding: .utf8)) + #expect(written.clusters.count == 1) + #expect(written.clusters[0].name == "dev") + #expect(written.currentContext == "dev") + } + + @Test func mergePreservesExistingExecAuthEntry() throws { + let (path, cleanup) = try makeTempFile() + defer { cleanup() } + + // Write an existing kubeconfig with a GKE exec-auth entry + let existingYAML = """ + apiVersion: v1 + kind: Config + clusters: + - name: gke-prod + cluster: + server: https://5.6.7.8 + certificate-authority-data: dGVzdA== + contexts: + - name: gke-prod + context: + cluster: gke-prod + user: gke-prod-user + namespace: production + current-context: gke-prod + users: + - name: gke-prod-user + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + command: gke-gcloud-auth-plugin + provideClusterInfo: true + interactiveMode: IfAvailable + """ + try existingYAML.write(toFile: path.string, atomically: true, encoding: .utf8) + + try K8sHelper.mergeConfig(makeConfig(clusterName: "dev"), containerId: "dev", targetPath: path, log: log) + + let result = try decode(String(contentsOfFile: path.string, encoding: .utf8)) + + // New entry present + #expect(result.clusters.contains { $0.name == "dev" }) + #expect(result.currentContext == "gke-prod") // existing context preserved — not overwritten + + // GKE entry fully preserved + let gkeCluster = try #require(result.clusters.first { $0.name == "gke-prod" }) + #expect(gkeCluster.cluster.server == "https://5.6.7.8") + + let gkeUser = try #require(result.users.first { $0.name == "gke-prod-user" }) + let exec = try #require(gkeUser.authInfo.exec) + #expect(exec.command == "gke-gcloud-auth-plugin") + #expect(exec.provideClusterInfo == true) + #expect(exec.interactiveMode == "IfAvailable") + + let gkeContext = try #require(result.contexts.first { $0.name == "gke-prod" }) + #expect(gkeContext.context.namespace == "production") + } + + @Test func mergePreservesTokenAuthEntry() throws { + let (path, cleanup) = try makeTempFile() + defer { cleanup() } + + let existingYAML = """ + apiVersion: v1 + kind: Config + clusters: + - name: staging + cluster: + server: https://9.10.11.12 + contexts: + - name: staging + context: + cluster: staging + user: staging-user + current-context: staging + users: + - name: staging-user + user: + token: verysecrettoken + """ + try existingYAML.write(toFile: path.string, atomically: true, encoding: .utf8) + + try K8sHelper.mergeConfig(makeConfig(clusterName: "dev"), containerId: "dev", targetPath: path, log: log) + + let result = try decode(String(contentsOfFile: path.string, encoding: .utf8)) + let stagingUser = try #require(result.users.first { $0.name == "staging-user" }) + #expect(stagingUser.authInfo.token == "verysecrettoken") + } + + @Test func mergeReplacesExistingEntryWithSameName() throws { + let (path, cleanup) = try makeTempFile() + defer { cleanup() } + + try K8sHelper.mergeConfig(makeConfig(clusterName: "dev", server: "https://127.0.0.1:6445"), containerId: "dev", targetPath: path, log: log) + try K8sHelper.mergeConfig(makeConfig(clusterName: "dev", server: "https://127.0.0.1:6446"), containerId: "dev", targetPath: path, log: log) + + let result = try decode(String(contentsOfFile: path.string, encoding: .utf8)) + #expect(result.clusters.filter { $0.name == "dev" }.count == 1) + #expect(result.contexts.filter { $0.name == "dev" }.count == 1) + #expect(result.users.filter { $0.name == "dev" }.count == 1) + #expect(result.clusters.first { $0.name == "dev" }?.cluster.server == "https://127.0.0.1:6446") + } + + @Test func mergeThrowsWhenExistingFileIsInvalidYAML() throws { + let (path, cleanup) = try makeTempFile() + defer { cleanup() } + + try "this: is: not: valid: yaml: [[[".write(toFile: path.string, atomically: true, encoding: .utf8) + + #expect(throws: (any Error).self) { + try K8sHelper.mergeConfig(makeConfig(clusterName: "dev"), containerId: "dev", targetPath: path, log: log) + } + } + + @Test func mergeSetsCurrentContextOnFirstClusterOnly() throws { + let (path, cleanup) = try makeTempFile() + defer { cleanup() } + + // First create: sets current-context because kubeconfig is empty + try K8sHelper.mergeConfig(makeConfig(clusterName: "first"), containerId: "first", targetPath: path, setCurrentContext: true, log: log) + // Second create: does NOT overwrite current-context because it's already set + try K8sHelper.mergeConfig(makeConfig(clusterName: "second"), containerId: "second", targetPath: path, setCurrentContext: true, log: log) + + let result = try decode(String(contentsOfFile: path.string, encoding: .utf8)) + #expect(result.currentContext == "first") // first-use semantics: not overwritten by second cluster + #expect(result.clusters.count == 2) + } +} + +// MARK: - env-dependent tests +// Both suites mutate the KUBECONFIG environment variable; wrap them in a +// common .serialized parent so they cannot race against each other. + +@Suite("K8sHelper env-dependent", .serialized) +struct KubeconfigEnvTests { + + // MARK: - resolveKubeconfigMergePath + + @Suite("K8sHelper.resolveKubeconfigMergePath") + struct ResolveKubeconfigMergePathTests { + + private func withKubeconfigEnv(_ value: String?, _ body: () -> Void) { + let key = "KUBECONFIG" + let original = Darwin.getenv(key).map { String(cString: $0) } + if let value { + setenv(key, value, 1) + } else { + unsetenv(key) + } + body() + if let original { + setenv(key, original, 1) + } else { + unsetenv(key) + } + } + + @Test func noEnvDefaultsToHomeKubeConfig() { + withKubeconfigEnv(nil) { + let path = K8sHelper.resolveKubeconfigMergePath() + #expect(path.string.hasSuffix(".kube/config")) + } + } + + @Test func singleEnvPathUsedDirectly() { + withKubeconfigEnv("/tmp/my-kubeconfig") { + let path = K8sHelper.resolveKubeconfigMergePath() + #expect(path.string == "/tmp/my-kubeconfig") + } + } + + @Test func multiplePathsFirstExistingWins() throws { + let existing = FilePath(FileManager.default.temporaryDirectory.path) + .appending("kube-exists-\(UUID().uuidString)") + try "".write(toFile: existing.string, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(atPath: existing.string) } + + withKubeconfigEnv("/tmp/does-not-exist-a:\(existing.string):/tmp/does-not-exist-b") { + let path = K8sHelper.resolveKubeconfigMergePath() + #expect(path.string == existing.string) + } + } + + @Test func multiplePathsNoneExistUsesLast() { + withKubeconfigEnv("/tmp/no-exist-a:/tmp/no-exist-b:/tmp/no-exist-c") { + let path = K8sHelper.resolveKubeconfigMergePath() + #expect(path.string == "/tmp/no-exist-c") + } + } + } + + // MARK: - removeConfig behavior + + @Suite("K8sHelper.removeConfig") + struct RemoveConfigTests { + + private func makeConfig(clusterName: String) -> KubeConfig { + var config = KubeConfig() + config.clusters = [NamedCluster(name: clusterName, cluster: Cluster(server: "https://127.0.0.1:6443"))] + config.contexts = [NamedContext(name: clusterName, context: Context(cluster: clusterName, user: clusterName))] + config.users = [NamedAuthInfo(name: clusterName, authInfo: AuthInfo())] + return config + } + + private func withKubeconfig(_ initial: KubeConfig, _ body: () throws -> Void) throws { + let tmp = FilePath(FileManager.default.temporaryDirectory.path) + .appending("kubeconfig-remove-\(UUID().uuidString)") + defer { try? FileManager.default.removeItem(atPath: tmp.string) } + try encode(initial).write(toFile: tmp.string, atomically: true, encoding: .utf8) + let original = Darwin.getenv("KUBECONFIG").map { String(cString: $0) } + setenv("KUBECONFIG", tmp.string, 1) + defer { + if let original { setenv("KUBECONFIG", original, 1) } else { unsetenv("KUBECONFIG") } + } + try body() + } + + private func currentKubeconfig() throws -> KubeConfig { + let path = K8sHelper.resolveKubeconfigMergePath() + return try decode(String(contentsOfFile: path.string, encoding: .utf8)) + } + + @Test func removesClusterContextAndUserEntries() throws { + var initial = makeConfig(clusterName: "dev") + let other = makeConfig(clusterName: "other") + initial.clusters += other.clusters + initial.contexts += other.contexts + initial.users += other.users + initial.currentContext = "other" + + try withKubeconfig(initial) { + try K8sHelper.removeConfig(containerId: "dev", log: log) + let result = try currentKubeconfig() + #expect(result.clusters.count == 1) + #expect(!result.clusters.contains { $0.name == "dev" }) + #expect(result.users.count == 1) + #expect(result.contexts.count == 1) + } + } + + @Test func clearsCurrentContextWhenItMatchesDeletedCluster() throws { + var initial = makeConfig(clusterName: "dev") + initial.currentContext = "dev" + + try withKubeconfig(initial) { + try K8sHelper.removeConfig(containerId: "dev", log: log) + #expect(try currentKubeconfig().currentContext == nil) + } + } + + @Test func preservesCurrentContextWhenItPointsElsewhere() throws { + var initial = makeConfig(clusterName: "dev") + let other = makeConfig(clusterName: "other") + initial.clusters += other.clusters + initial.contexts += other.contexts + initial.users += other.users + initial.currentContext = "other" + + try withKubeconfig(initial) { + try K8sHelper.removeConfig(containerId: "dev", log: log) + #expect(try currentKubeconfig().currentContext == "other") + } + } + + @Test func noopWhenFileDoesNotExist() throws { + let missing = FilePath(FileManager.default.temporaryDirectory.path) + .appending("kubeconfig-missing-\(UUID().uuidString)") + let original = Darwin.getenv("KUBECONFIG").map { String(cString: $0) } + setenv("KUBECONFIG", missing.string, 1) + defer { + if let original { setenv("KUBECONFIG", original, 1) } else { unsetenv("KUBECONFIG") } + } + try K8sHelper.removeConfig(containerId: "dev", log: log) + } + } +} diff --git a/docs/command-reference.md b/docs/command-reference.md index 78a1835e7..7a9a3a04d 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -1583,3 +1583,175 @@ container system property list # output as JSON for scripting container system property list --format json ``` + +## Kubernetes Cluster Management + +`container k8s` manages local single-node Kubernetes clusters backed by container VMs. Each cluster runs a Kubernetes control-plane node inside a container using `kindest/node` and `kubeadm`. + +> [!IMPORTANT] +> The `k8s` command is an experimental feature and its subcommands and options are subject to change. + +### `container k8s create` + +Creates and starts a local Kubernetes cluster. Pulls the node image if needed, runs `kubeadm init`, installs the kindnet CNI, and merges the cluster credentials into `~/.kube/config`. + +**Usage** + +```bash +container k8s create [--name ] [--node-image ] [--rm] [] [--debug] +``` + +**Options** + +* `--name `: Cluster name (default: `k8s-dev`) +* `--node-image `: Node image reference (default: `docker.io/kindest/node:v1.35.5`) +* `--rm`: Remove the cluster container after it stops + +**Resource Options** + +* `--cpus `: Number of virtual CPUs (default: 1/4 of host CPUs, minimum 2) +* `--memory `: Memory allocation (default: 1/4 of host memory, minimum 2g) + +**Registry Options** + +* `--scheme `: Scheme for the container registry (values: http, https, auto; default: auto) + +**Image Fetch Options** + +* `--max-concurrent-downloads `: Maximum number of concurrent downloads (default: 3) + +**Examples** + +```bash +# create a cluster with the default name (k8s-dev) +container k8s create + +# create a cluster with a custom name and resource allocation +container k8s create --name my-cluster --cpus 4 --memory 8g + +# create a cluster that removes itself when stopped +container k8s create --name temp-cluster --rm +``` + +### `container k8s start` + +Starts a stopped Kubernetes cluster and refreshes its entry in `~/.kube/config` (the container IP can change between starts). + +**Usage** + +```bash +container k8s start [--name ] [--debug] +``` + +**Options** + +* `--name `: Cluster name (default: `k8s-dev`) + +**Examples** + +```bash +# start the default cluster +container k8s start + +# start a named cluster +container k8s start --name my-cluster +``` + +### `container k8s delete (rm)` + +Stops and deletes a Kubernetes cluster container and removes its entry from `~/.kube/config`. + +**Usage** + +```bash +container k8s delete [--name ] [--debug] +``` + +**Options** + +* `--name `: Cluster name (default: `k8s-dev`) + +**Examples** + +```bash +# delete the default cluster +container k8s delete + +# delete a named cluster +container k8s delete --name my-cluster +container k8s rm --name my-cluster +``` + +### `container k8s list (ls)` + +Lists all Kubernetes clusters with their status and node image. + +**Usage** + +```bash +container k8s list [--debug] +``` + +**Examples** + +```bash +container k8s list +container k8s ls +``` + +### `container k8s load-image` + +Exports an image from the local `container` image store and imports it into the cluster's containerd (in the `k8s.io` namespace) so that Kubernetes can schedule pods that reference it. + +**Usage** + +```bash +container k8s load-image [--name ] [--platform ] [--debug] +``` + +**Arguments** + +* ``: Image reference to load (e.g. `my-app:latest`) + +**Options** + +* `--name `: Cluster name (default: `k8s-dev`) +* `--platform `: Platform of the image variant to load from a multi-arch image (format: os/arch[/variant], default: `linux/`). Use this when the local store contains a multi-arch manifest list and you want to select a specific variant. + +**Examples** + +```bash +# load an image into the default cluster +container k8s load-image my-app:latest + +# load an image into a named cluster +container k8s load-image --name my-cluster my-app:latest + +# load the amd64 variant of a multi-arch image +container k8s load-image --platform linux/amd64 my-app:latest +``` + +### `container k8s write-config` + +Fetches the current kubeconfig from a running cluster and merges its context into a kubeconfig file. Use this to refresh credentials after a cluster restart or to write to an alternate config file. + +**Usage** + +```bash +container k8s write-config [--name ] [--kubeconfig ] [--debug] +``` + +**Options** + +* `--name `: Cluster name (default: `k8s-dev`) +* `--kubeconfig `: Path to the kubeconfig file to write or append to (default: `~/.kube/config`) + +**Examples** + +```bash +# refresh credentials for the default cluster into ~/.kube/config +container k8s write-config + +# write the context for a named cluster to an alternate kubeconfig file +container k8s write-config --name my-cluster --kubeconfig ~/.kube/my-cluster.kubeconfig +```