diff --git a/.github/linters/urunc-dict.txt b/.github/linters/urunc-dict.txt index 933cca0b1..0c867b64c 100644 --- a/.github/linters/urunc-dict.txt +++ b/.github/linters/urunc-dict.txt @@ -433,3 +433,9 @@ hyperlight Hyperlight Odysseas Kalaitsidis +nsexec +specconv +Readonlyfs +capab +werr +cerr diff --git a/.github/workflows/kind_test.yml b/.github/workflows/kind_test.yml index 43719e70c..c77f757de 100644 --- a/.github/workflows/kind_test.yml +++ b/.github/workflows/kind_test.yml @@ -335,6 +335,48 @@ jobs: grep "Hello world" /tmp/logs.txt kubectl describe pod hello-spt-rumprun-block || true + # Re-run the hello-world deployment with urunc's libcontainer runtime + # enabled, reusing the same kind cluster and urunc install. + - name: Deploy and verify with libcontainer + run: | + docker exec urunc-test-control-plane bash -c ' + cat >> /etc/urunc/config.toml < /dev/null <<'EOF' + + [runtime] + libcontainer = true + EOF + export GOROOT=$(go env GOROOT) + export PATH="$GOROOT/bin:$PATH" + if [ "${{ matrix.arch }}" = "arm64" ]; then + sudo -E env "PATH=$PATH" "GOROOT=$GOROOT" make ${{ matrix.test }}_Spt + else + sudo -E env "PATH=$PATH" "GOROOT=$GOROOT" make ${{ matrix.test }} SKIP="Firecracker-unikraft-httpreply-static-net" + fi + - name: Dump urunc logs on failure if: failure() run: | diff --git a/Makefile b/Makefile index b0329d2dc..42a0bbb7d 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,10 @@ CGO := CGO_ENABLED=1 NOCGO := CGO_ENABLED=0 TEST_FLAGS := "-count=1" TEST_OPTS += -timeout 20m +# Skip the execution of a test +# make test_crictl SKIP="Firecracker-unikraft-httpreply-static-net" +# make test_crictl SKIP="Crictl.*(CaseA|CaseB)" +GINKGO_SKIP := $(if $(SKIP),--ginkgo.skip="$(SKIP)") BUILD_TAGS ?= netgo osusergo # Linking variables @@ -273,56 +277,56 @@ test_unikernels: .PHONY: test_nerdctl test_nerdctl: @echo "Testing nerdctl" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Nerdctl" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Nerdctl" $(GINKGO_SKIP) @echo " " ## test_ctr Run all end-to-end tests with ctr .PHONY: test_ctr test_ctr: @echo "Testing ctr" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Ctr" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Ctr" $(GINKGO_SKIP) @echo " " ## test_crictl Run all end-to-end tests with crictl .PHONY: test_crictl test_crictl: @echo "Testing crictl" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Crictl" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Crictl" $(GINKGO_SKIP) @echo " " ## test_docker Run all end-to-end tests with docker .PHONY: test_docker test_docker: @echo "Testing docker" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Docker" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -run TestE2E -v --ginkgo.v --ginkgo.focus="Docker" $(GINKGO_SKIP) @echo " " ## test_nerdctl_[pattern] Run all end-to-end tests with nerdctl that match pattern .PHONY: test_nerdctl_% test_nerdctl_%: @echo "Testing nerdctl" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Nerdctl.*$*" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Nerdctl.*$*" $(GINKGO_SKIP) @echo " " ## test_ctr_[pattern] Run all end-to-end tests with ctr that match pattern .PHONY: test_ctr_% test_ctr_%: @echo "Testing ctr" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Ctr.*$*" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Ctr.*$*" $(GINKGO_SKIP) @echo " " ## test_crictl_[pattern] Run all end-to-end tests with crictl that match pattern .PHONY: test_crictl_% test_crictl_%: @echo "Testing crictl" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Crictl.*$*" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Crictl.*$*" $(GINKGO_SKIP) @echo " " ## test_docker_[pattern] Run all end-to-end tests with docker that match pattern .PHONY: test_docker_% test_docker_%: @echo "Testing docker" - @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Docker.*$*" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./tests/e2e -v --ginkgo.v -run TestE2E --ginkgo.focus="Docker.*$*" $(GINKGO_SKIP) @echo " " ## help Show this help message diff --git a/cmd/urunc/create.go b/cmd/urunc/create.go index 5a9e2f94d..14bbffea1 100644 --- a/cmd/urunc/create.go +++ b/cmd/urunc/create.go @@ -73,25 +73,28 @@ var createCommand = &cli.Command{ if err := checkArgs(cmd, 1, exactArgs); err != nil { return err } - if !cmd.Bool("reexec") { - uruncCfg, _ := unikontainers.LoadUruncConfig(unikontainers.UruncConfigPath) // ignore the error and use default config - return createUnikontainer(cmd, uruncCfg) + if cmd.Bool("reexec") { + return reexecUnikontainer(cmd) } - return reexecUnikontainer(cmd) + uruncCfg, _ := unikontainers.LoadUruncConfig(unikontainers.UruncConfigPath) // ignore the error and use default config + if uruncCfg.Runtime.Libcontainer { + return libcontainerCreate(cmd, uruncCfg) + } + + return createUnikontainer(cmd, uruncCfg) }, } -// createUnikontainer creates a Unikernel struct from bundle data, -// initializes it's base dir and state.json, -// setups terminal if required and spawns reexec process, -// waits for reexec process to notify, executes CreateRuntime hooks, -// sends ACK to reexec process -func createUnikontainer(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) (err error) { - err = nil +// newUnikontainer parses the bundle and performs the host-side preparation for +// the monitor execution environment (Unikontainer, base directory, state and +// monitor resources). It never returns for a container that is not a urunc +// container: those are handed over to the real runc with an execve. +func newUnikontainer(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) (*unikontainers.Unikontainer, error) { containerID := cmd.Args().First() - if err = validateID(containerID); err != nil { - return err + err := validateID(containerID) + if err != nil { + return nil, err } metrics.SetLoggerContainerID(containerID) metrics.Capture(m.TS00) @@ -105,7 +108,7 @@ func createUnikontainer(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) ( if bundlePath == "" { bundlePath, err = os.Getwd() if err != nil { - return err + return nil, err } } @@ -114,21 +117,33 @@ func createUnikontainer(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) ( if err != nil { if errors.Is(err, unikontainers.ErrQueueProxy) || errors.Is(err, unikontainers.ErrNotUnikernel) { - // Exec runc to handle non unikernel containers - err = runcExec() - return err + // Exec runc to handle non urunc containers. + // It should never return. + return nil, runcExec() } - return err + return nil, err } metrics.Capture(m.TS01) err = unikontainer.InitialSetup() if err != nil { - return err + return nil, err } - metrics.Capture(m.TS02) + return unikontainer, nil +} + +// createUnikontainer creates a Unikernel struct from bundle data, initializes +// it's base dir and state.json, setups terminal if required and spawns reexec +// process, waits for reexec process to notify, executes CreateRuntime hooks, +// sends ACK to reexec process +func createUnikontainer(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) (err error) { + unikontainer, err := newUnikontainer(cmd, uruncCfg) + if err != nil { + return err + } + // Create socket for nsenter initSockParent, initSockChild, err := newSockPair("init") if err != nil { diff --git a/cmd/urunc/create_libcontainer.go b/cmd/urunc/create_libcontainer.go new file mode 100644 index 000000000..8778387a4 --- /dev/null +++ b/cmd/urunc/create_libcontainer.go @@ -0,0 +1,188 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "net" + "os" + "strconv" + + "github.com/opencontainers/runc/libcontainer" + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v3" + m "github.com/urunc-dev/urunc/internal/metrics" + "github.com/urunc-dev/urunc/pkg/unikontainers" +) + +// monitorArgv is the internal argv of the urunc monitor process. +const monitorArgv = "monitor" + +// libcontainerCreate is "urunc create" when the libcontainer runtime is +// enabled. It configures libcontainer to spawn a specific urunc process +// ("urunc monitor") which will finalize the execution environment for a +// monitor and then execve it. +func libcontainerCreate(cmd *cli.Command, uruncCfg *unikontainers.UruncConfig) (err error) { + unikontainer, err := newUnikontainer(cmd, uruncCfg) + if err != nil { + return err + } + + config, err := unikontainer.BuildContainerConfig(cmd.Bool("systemd-cgroup")) + if err != nil { + return err + } + + container, err := libcontainer.Create(unikontainer.LibcontainerRoot(), unikontainer.State.ID, config) + if err != nil { + return fmt.Errorf("failed to create the monitor's container: %w", err) + } + defer func() { + // Best effort cleanup in case of failure. Any cgroups or libcontainer + // metadata residuals can cause issues with later container creates. + if err != nil { + tmpErr := container.Destroy() + if tmpErr != nil { + logrus.WithError(tmpErr).Error("failed to destroy the monitor's container") + } + } + }() + + // The write end of the ready pipe. The monitor process inherits the + // write end of the ready pipe and reports the outcome of its setup + // back to "urunc start" over it. + + readyFile, err := unikontainers.CreateReadyPipe(unikontainer.BaseDir) + if err != nil { + return err + } + defer readyFile.Close() + + process, err := monitorProcess(cmd, unikontainer, readyFile) + if err != nil { + return err + } + metrics.Capture(m.TS03) + + err = container.Start(process) + if err != nil { + return fmt.Errorf("failed to start the monitor's init process: %w", err) + } + // The init process is now waiting the exec fifo. If anything below fails, + // terminate and reap it before the Destroy above tears down the cgroup, so the + // reap does not race the cgroup removal (same order as runc). + defer func() { + if err != nil { + _ = process.Signal(os.Kill) + _, _ = process.Wait() + } + }() + + state, err := container.State() + if err != nil { + return fmt.Errorf("failed to read the monitor's container state: %w", err) + } + metrics.Capture(m.TS06) + + err = unikontainer.Create(state.InitProcessPid, cmd.String("pid-file")) + if err != nil { + return err + } + metrics.Capture(m.TS08) + + return nil +} + +// buildMonitorArgs assembles the "urunc monitor" argv setting the same log format +// and enabling debug logs based on the current urunc process. +// TODO: THe logs will be printed in stdout. We need to pass another fd with the logs +// in order to store them in the correct file. +func buildMonitorArgs(id string, logFormat string) []string { + args := []string{"/proc/self/exe"} + logLevel := logrus.GetLevel() + if logLevel >= logrus.DebugLevel { + // TODO: We need to pass the log level not just debug. + // However, this needs to change the cli args of urunc. + // So let's do it in future iteration. + args = append(args, "--debug") + } + + if logFormat != "" { + args = append(args, "--log-format", logFormat) + } + + args = append(args, monitorArgv, id) + + return args +} + +// monitorProcess describes the urunc process that libcontainer starts. +// Args[0] is "/proc/self/exe" because libcontainer resolves it after the +// pivot: /proc is always mounted in the monitor rootfs, so this re-execs the +// sealed urunc binary without urunc having to be copied into that rootfs. +func monitorProcess(cmd *cli.Command, u *unikontainers.Unikontainer, readyFile *os.File) (*libcontainer.Process, error) { + process := &libcontainer.Process{ + Args: buildMonitorArgs(u.State.ID, cmd.String("log-format")), + Env: os.Environ(), + // "/" rather than the spec's Cwd: that one is the guest's working + // directory + Cwd: "/", + // TODO set uid, gid and additional groups here + Init: true, + Capabilities: unikontainers.MonitorCapabilities(u.Spec), + LogLevel: strconv.Itoa(int(logrus.GetLevel())), + // The ready pipe's write end. It must be the only/first ExtraFile so the + // monitor inherits it at unikontainers.ReadyPipeFD: + ExtraFiles: []*os.File{readyFile}, + } + + if !u.Spec.Process.Terminal { + process.Stdin = os.Stdin + process.Stdout = os.Stdout + process.Stderr = os.Stderr + + return process, nil + } + + // With a terminal, libcontainer's init allocates the pty and passes + // the master end over the console socket itself The console socket is + // an AF_UNIX socket the caller (e.g. containerd) is listening on, so + // it must be dialed, not opened as a file: open(2) on a socket inode + // fails with ENXIO. + consoleSocket := cmd.String("console-socket") + if consoleSocket == "" { + return nil, fmt.Errorf("the container requests a terminal but no console socket was given") + } + conn, err := net.Dial("unix", consoleSocket) + if err != nil { + return nil, fmt.Errorf("failed to dial the console socket %s: %w", consoleSocket, err) + } + defer conn.Close() + + unixConn, ok := conn.(*net.UnixConn) + if !ok { + return nil, fmt.Errorf("console socket connection is not a unix socket") + } + // File returns a dup of the socket fd, independent of conn, so closing conn + // above does not affect it. libcontainer passes it to its init as an extra + // file and sends the pty master over it. + socket, err := unixConn.File() + if err != nil { + return nil, fmt.Errorf("failed to get the console socket file: %w", err) + } + process.ConsoleSocket = socket + + return process, nil +} diff --git a/cmd/urunc/init.go b/cmd/urunc/init.go new file mode 100644 index 000000000..a32e5e24c --- /dev/null +++ b/cmd/urunc/init.go @@ -0,0 +1,33 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// File taken from https://github.com/opencontainers/runc v1.5.1 Please +// check the license there too. +package main + +import ( + "os" + + "github.com/opencontainers/runc/libcontainer" + _ "github.com/opencontainers/runc/libcontainer/nsenter" +) + +// Taken from https://github.com/opencontainers/runc/blob/v1.5.1/init.go +func init() { + if len(os.Args) > 1 && os.Args[1] == "init" { + // This is the golang entry point for runc init, executed + // before main() but after libcontainer/nsenter's nsexec(). + libcontainer.Init() + } +} diff --git a/cmd/urunc/main.go b/cmd/urunc/main.go index 7a103aacf..fdbcda979 100644 --- a/cmd/urunc/main.go +++ b/cmd/urunc/main.go @@ -30,7 +30,6 @@ import ( m "github.com/urunc-dev/urunc/internal/metrics" "github.com/urunc-dev/urunc/pkg/unikontainers" - _ "github.com/opencontainers/runc/libcontainer/nsenter" "github.com/urfave/cli/v3" ) @@ -112,6 +111,7 @@ func main() { createCommand, deleteCommand, killCommand, + monitorCommand, runCommand, psCommand, // specCommand, @@ -119,20 +119,27 @@ func main() { // stateCommand, }, Before: func(_ context.Context, cmd *cli.Command) (context.Context, error) { - if !cmd.IsSet("root") { - xdgRuntimeDir := os.Getenv("XDG_RUNTIME_DIR") - if xdgRuntimeDir != "" && ShouldHonorXDGRuntimeDir() { - root := xdgRuntimeDir + "/urunc" - if err := prepareXDGRuntimeDir(root); err != nil { - return nil, err - } - if err := cmd.Set("root", root); err != nil { - return nil, err + // "urunc monitor" runs inside the monitor rootfs and + // never touches urunc's state directory, so it has no + // root to resolve. Also the inherited XDG_RUNTIME_DIR + // names a directory that does not exist in there, so + // preparing it would fail the monitor. + if !isMonitorInvocation(cmd) { + if !cmd.IsSet("root") { + xdgRuntimeDir := os.Getenv("XDG_RUNTIME_DIR") + if xdgRuntimeDir != "" && ShouldHonorXDGRuntimeDir() { + root := xdgRuntimeDir + "/urunc" + if err := prepareXDGRuntimeDir(root); err != nil { + return nil, err + } + if err := cmd.Set("root", root); err != nil { + return nil, err + } } } - } - if err := reviseRootDir(cmd); err != nil { - return nil, err + if err := reviseRootDir(cmd); err != nil { + return nil, err + } } // ignore error since ParseLogMetricsConfig will print a warning and return default values cfg, _ := unikontainers.ParseLogMetricsConfig(unikontainers.UruncConfigPath) diff --git a/cmd/urunc/monitor.go b/cmd/urunc/monitor.go new file mode 100644 index 000000000..23204ace9 --- /dev/null +++ b/cmd/urunc/monitor.go @@ -0,0 +1,57 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "os" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v3" + "github.com/urunc-dev/urunc/pkg/unikontainers" +) + +// monitorCommand is not part of the OCI runtime interface and no user or shim +// ever calls it: it is the argv libcontainer's init execve's inside the +// monitor rootfs. By that point the vast majority of the execution environment +// (namespaces, cgroups, mounts, etc.) are all in place and all +// that is left is to create the tap device and become the monitor. +// It takes the container ID purely so that the logs and the process listing name +// the container; everything it actually needs comes from the monitor spec file that +// create wrote inside the monitor's rootfs. +var monitorCommand = &cli.Command{ + Name: monitorArgv, + Usage: "internal: launch the monitor from inside its own rootfs", + ArgsUsage: ``, + Hidden: true, + Action: func(_ context.Context, cmd *cli.Command) error { + logrus.WithField("command", "MONITOR").WithField("args", os.Args).Debug("urunc INVOKED") + err := checkArgs(cmd, 1, exactArgs) + if err != nil { + return err + } + metrics.SetLoggerContainerID(cmd.Args().First()) + + // ExecMonitor does not return on success: the monitor replaces this + // process and becomes the container process. + return unikontainers.ExecMonitor(metrics) + }, +} + +// isMonitorInvocation reports whether this process is the "urunc monitor" that +// libcontainer starts inside the monitor container. +func isMonitorInvocation(cmd *cli.Command) bool { + return cmd.Args().First() == monitorArgv +} diff --git a/cmd/urunc/start.go b/cmd/urunc/start.go index a020b5471..51e15dd45 100644 --- a/cmd/urunc/start.go +++ b/cmd/urunc/start.go @@ -20,6 +20,7 @@ import ( "fmt" "os" + "github.com/opencontainers/runc/libcontainer" "github.com/sirupsen/logrus" "github.com/urfave/cli/v3" m "github.com/urunc-dev/urunc/internal/metrics" @@ -70,38 +71,27 @@ func startUnikontainer(cmd *cli.Command) error { defer func() { tmpErr := unikontainer.DestroyListener(!unikontainers.FromReexec) if tmpErr != nil { - logrus.WithError(tmpErr).Error("failed to destroy listener on reexec socket") + logrus.WithError(tmpErr).Error("failed to destroy the start listener") } }() - // Send message to reexec to start the monitor - err = unikontainer.CreateConn(!unikontainers.FromReexec) - if err != nil { - err = fmt.Errorf("failed to create connection with reexec socket: %w", err) - return err - } - sendErr := unikontainer.SendMessage(unikontainers.StartExecve) - if sendErr != nil { - logrus.WithError(sendErr).Error("failed to send START message to reexec") - sendErr = fmt.Errorf("error sending START message: %w", sendErr) - } - // Regardless of the SendMessage status, make sure to clean up the socket, - // since it is not required anymore - cleanErr := unikontainer.DestroyConn(!unikontainers.FromReexec) - if cleanErr != nil { - logrus.WithError(cleanErr).Error("failed to destroy connection to reexec socket") - cleanErr = fmt.Errorf("error destroying connection to reexec socket: %w", cleanErr) + // Start the monitor environment setup process. For libcontainer the heavy lifting + // has be done from libcontainer and we just need to finalize the setup. + // For urunc native way, we need to setup the monitor rootfs too. + if unikontainer.UruncCfg.Runtime.Libcontainer { + err = startMonitorInit(unikontainer) + } else { + err = startReexec(unikontainer) } - err = errors.Join(sendErr, cleanErr) if err != nil { return err } metrics.Capture(m.TS13) - // wait ContainerStarted message on start.sock from reexec process + // Wait for the monitor process to report a successful start. err = unikontainer.AwaitMsg(unikontainers.StartSuccess) if err != nil { - err = fmt.Errorf("failed to get message from successful start from reexec: %w", err) + err = fmt.Errorf("failed to get the success message from the monitor: %w", err) return err } @@ -113,3 +103,42 @@ func startUnikontainer(cmd *cli.Command) error { return unikontainer.ExecuteHooks("Poststart") } + +// startMonitorInit starts the urunc monitor process, blocked since create from +// libcontainers, by opening the exec fifo. +func startMonitorInit(unikontainer *unikontainers.Unikontainer) error { + container, err := libcontainer.Load(unikontainer.LibcontainerRoot(), unikontainer.State.ID) + if err != nil { + return fmt.Errorf("failed to load the monitor's container: %w", err) + } + + err = container.Exec() + if err != nil { + return fmt.Errorf("failed to release the monitor process: %w", err) + } + + return nil +} + +// startReexec tells urunc's reexec process to set up the monitor's +// environment and launch it. +func startReexec(unikontainer *unikontainers.Unikontainer) error { + err := unikontainer.CreateConn(!unikontainers.FromReexec) + if err != nil { + return fmt.Errorf("failed to create connection with reexec socket: %w", err) + } + sendErr := unikontainer.SendMessage(unikontainers.StartExecve) + if sendErr != nil { + logrus.WithError(sendErr).Error("failed to send START message to reexec") + sendErr = fmt.Errorf("error sending START message: %w", sendErr) + } + // Regardless of the SendMessage status, make sure to clean up the socket, + // since it is not required anymore + cleanErr := unikontainer.DestroyConn(!unikontainers.FromReexec) + if cleanErr != nil { + logrus.WithError(cleanErr).Error("failed to destroy connection to reexec socket") + cleanErr = fmt.Errorf("error destroying connection to reexec socket: %w", cleanErr) + } + + return errors.Join(sendErr, cleanErr) +} diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba4..6e3d65a7a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,6 +9,9 @@ The configuration file uses the [TOML](https://toml.io/) format and is organized into several sections: ```toml +[runtime] +libcontainer = false + [log] level = "info" syslog = false @@ -42,6 +45,29 @@ options = "--sandbox none" ## Configuration Sections +### Runtime + +The `[runtime]` section controls runtime-wide behavior. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `libcontainer` | boolean | `false` | Use libcontainer to set up the monitor's execution environment | + +> **⚠️ Experimental:** the use of `libcontainer` to prepare the monitor execution +> environment is under active development. It is off by default. + +The value is resolved when the container is created and recorded in the +container's `state.json`, so `start`, `kill` and `delete` keep using the mode +the container was created with even if the configuration file changes in +between. + +**Example:** + +```toml +[runtime] +libcontainer = false +``` + ### Log Configuration The `[log]` section controls logging behavior for `urunc`: diff --git a/go.mod b/go.mod index 526ab367d..161c7b140 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/nubificus/hedge_cli v0.0.3 github.com/onsi/ginkgo/v2 v2.32.0 github.com/onsi/gomega v1.42.1 + github.com/opencontainers/cgroups v0.0.4 github.com/opencontainers/image-spec v1.1.1 github.com/opencontainers/runc v1.3.6 github.com/opencontainers/runtime-spec v1.2.1 @@ -36,10 +37,12 @@ require ( ) require ( + cyphar.com/go-pathrs v0.2.5 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.13.0 // indirect - github.com/cilium/ebpf v0.21.0 // indirect + github.com/checkpoint-restore/go-criu/v6 v6.3.0 // indirect + github.com/cilium/ebpf v0.17.3 // indirect github.com/containerd/cgroups/v3 v3.1.0 // indirect github.com/containerd/console v1.0.5 // indirect github.com/containerd/continuity v0.5.0 // indirect @@ -63,11 +66,15 @@ require ( github.com/klauspost/compress v1.18.6 // indirect github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.22 // indirect + github.com/moby/sys/capability v0.4.0 // indirect github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/user v0.4.0 // indirect + github.com/mrunalp/fileutils v0.5.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/selinux v1.13.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/seccomp/libseccomp-golang v0.10.0 // indirect go.opencensus.io v0.24.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/mod v0.36.0 // indirect diff --git a/go.sum b/go.sum index bee2d7796..027790197 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v6 v6.3.0 h1:mIdrSO2cPNWQY1truPg6uHLXyKHk3Z5Odx4wjKOASzA= github.com/checkpoint-restore/go-criu/v6 v6.3.0/go.mod h1:rrRTN/uSwY2X+BPRl/gkulo9gsKOSAeVp9/K2tv7xZI= -github.com/cilium/ebpf v0.21.0 h1:4dpx1J/B/1apeTmWBH5BkVLayHTkFrMovVPnHEk+l3k= -github.com/cilium/ebpf v0.21.0/go.mod h1:1kHKv6Kvh5a6TePP5vvvoMa1bclRyzUXELSs272fmIQ= +github.com/cilium/ebpf v0.17.3 h1:FnP4r16PWYSE4ux6zN+//jMcW4nMVRvuTLVTvCjyyjg= +github.com/cilium/ebpf v0.17.3/go.mod h1:G5EDHij8yiLzaqn0WjyfJHvRa+3aDlReIaLVRMvOyJk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/cgroups/v3 v3.1.0 h1:azxYVj+91ZgSnIBp2eI3k9y2iYQSR/ZQIgh9vKO+HSY= @@ -52,6 +52,7 @@ github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwl github.com/containerd/typeurl/v2 v2.3.0/go.mod h1:Qk+PAdUYArVj41TnGi6rJ+48RF0PkcTc4i/taoBcK0w= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/cyphar/filepath-securejoin v0.7.0 h1:s0Y3ITPy6sQn5xt54DuYvTF8hu134ooYLUb58DX/HjE= @@ -80,8 +81,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s= -github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= @@ -102,6 +103,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -110,6 +112,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= @@ -119,6 +122,7 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jackpal/gateway v1.2.0 h1:euPRe4t7JfTaqC5Lr78HXl2wSHo54XndTtiAcIxkb5g= github.com/jackpal/gateway v1.2.0/go.mod h1:/jchvRi4HukAqV24da70iaBMFcSrX3rNWdR5K9VHd0A= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= @@ -141,8 +145,8 @@ github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= -github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= -github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= +github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= +github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk= @@ -190,11 +194,14 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY= github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -266,6 +273,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -310,11 +318,14 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/unikontainers/block.go b/pkg/unikontainers/block.go index 69db4629a..aaac2446b 100644 --- a/pkg/unikontainers/block.go +++ b/pkg/unikontainers/block.go @@ -426,7 +426,7 @@ func (b blockRootfs) getSharedDirs() (types.SharedfsParams, error) { return types.SharedfsParams{}, nil } -func (b blockRootfs) preStart() error { +func (b blockRootfs) preStartCmd() []string { return nil } diff --git a/pkg/unikontainers/initrd_rootfs.go b/pkg/unikontainers/initrd_rootfs.go index ee70d9da2..136b77365 100644 --- a/pkg/unikontainers/initrd_rootfs.go +++ b/pkg/unikontainers/initrd_rootfs.go @@ -57,6 +57,6 @@ func (i initrdRootfs) getSharedDirs() (types.SharedfsParams, error) { return types.SharedfsParams{}, nil } -func (i initrdRootfs) preStart() error { +func (i initrdRootfs) preStartCmd() []string { return nil } diff --git a/pkg/unikontainers/ipc.go b/pkg/unikontainers/ipc.go index 86d54e4dc..2fc7327d1 100644 --- a/pkg/unikontainers/ipc.go +++ b/pkg/unikontainers/ipc.go @@ -17,10 +17,12 @@ package unikontainers import ( "errors" "fmt" + "io" "io/fs" "net" "os" "path/filepath" + "syscall" "time" "github.com/sirupsen/logrus" @@ -29,6 +31,14 @@ import ( type IPCMessage string const ( + // readyPipeName is the FIFO in the container's state dir (BaseDir). + readyPipeName = "urunc-ready.fifo" + // readyOK is the byte the urunc monitor process writes on a successful setup. + readyOK byte = 0 + // ReadyPipeFD is the descriptor the FIFO's write end lands on inside + // the urunc monitor process.The ready pipe is passed as the single, + // first ExtraFile so it is always fd 3. All other files are placed after it. + ReadyPipeFD = 3 // Socket for messages towards reexec. The reexec process listens in this socket reexecSock = "reexec.sock" // Socket for messages from reexec. The reexec process writes in this socket @@ -55,6 +65,10 @@ func getReexecSockAddr(baseDir string) string { return getSockAddr(baseDir, reexecSock) } +func getReadyPipePath(baseDir string) string { + return filepath.Join(baseDir, readyPipeName) +} + func ensureValidSockAddr(sockAddr string) error { if sockAddr == "" { return fmt.Errorf("socket address is empty") @@ -169,3 +183,103 @@ func AwaitMessage(listener *net.UnixListener, expectedMessage IPCMessage) error } return nil } + +// CreateReadyPipe creates the ready FIFO in the container's state dir and +// opens its write end. The returned file is meant to be passed as the urunc's +// monitor process' single Process.ExtraFiles entry, so the monitor inherits it +// at ReadyPipeFD. +func CreateReadyPipe(baseDir string) (*os.File, error) { + path := getReadyPipePath(baseDir) + + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("failed to remove stale ready pipe %s: %w", path, err) + } + + err = syscall.Mkfifo(path, 0o600) + if err != nil { + return nil, fmt.Errorf("failed to create ready pipe %s: %w", path, err) + } + + // Open O_RDWR, which never blocks on a FIFO and, more importantly, + // keeps a write end open from the moment the monitor is born. That way + // the FIFO always "has a writer" while the monitor runs, so "urunc + // start" blocks reading it rather than seeing a premature EOF, yet it + // still reports EOF if the monitor dies without writing. + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return nil, fmt.Errorf("failed to open ready pipe %s: %w", path, err) + } + + return file, nil +} + +// openReadReadyPipe opens the read end of the ready FIFO that "urunc create" left in +// the state dir. +func (u *Unikontainer) openReadReadyPipe() error { + path := getReadyPipePath(u.BaseDir) + + // Open as non-blocking so the open never waits on a writer; + // the actual read in awaitReady blocks through the Go runtime poller. + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return fmt.Errorf("failed to open ready pipe %s: %w", path, err) + } + u.readyPipe = os.NewFile(uintptr(fd), path) + + return nil +} + +// closeReadReadyPipe closes the read end of the ready FIFO and removes it. +func (u *Unikontainer) closeReadReadyPipe() error { + if u.readyPipe != nil { + err := u.readyPipe.Close() + if err != nil { + uniklog.WithError(err).Error("failed to close the ready pipe") + } + } + + path := getReadyPipePath(u.BaseDir) + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to remove ready pipe %s: %w", path, err) + } + + return nil +} + +// awaitReadyPipe blocks until the urunc monitor process reports the outcome of its setup +// over the ready FIFO. A single readyOK byte means success; any other byte, a read +// error, or EOF (the monitor exited before reporting) means failure. +func (u *Unikontainer) awaitReadyPipe() error { + buf := make([]byte, 1) + + n, err := u.readyPipe.Read(buf) + if err != nil { + if errors.Is(err, io.EOF) { + return fmt.Errorf("the monitor exited before reporting a successful start") + } + return fmt.Errorf("failed to read from the ready pipe: %w", err) + } + // buf is make([]byte, 1); index 0 is a valid value + if n != 1 || buf[0] != readyOK { //nolint:gosec + return fmt.Errorf("the monitor reported a failed start") + } + + return nil +} + +// signalReady reports the outcome of the monitor setup to "urunc start" over the +// ready FIFO inherited, then closes it so it is not left open in the +// monitor. +func signalReady(ok bool) error { + status := byte(1) + if ok { + status = readyOK + } + + _, werr := syscall.Write(ReadyPipeFD, []byte{status}) + cerr := syscall.Close(ReadyPipeFD) + + return errors.Join(werr, cerr) +} diff --git a/pkg/unikontainers/libcontainer.go b/pkg/unikontainers/libcontainer.go new file mode 100644 index 000000000..93d0cc8a0 --- /dev/null +++ b/pkg/unikontainers/libcontainer.go @@ -0,0 +1,99 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unikontainers + +import ( + "slices" + + "github.com/opencontainers/runc/libcontainer/configs" + "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// monCaps all the necessary capabilities for the "urunc init" process +// to finalize the setup of the monitor's execution environment. +// - CAP_NET_ADMIN: required for the creation and setup of th the tap device +// inside the container's network namespace. +var monCaps = []string{"CAP_NET_ADMIN"} + +// monitorOCISpec builds the OCI spec that describes the monitor's +// execution environment. It consists of the container's spec, +// but altered accordingly for the respective monitor and rootfs. +func monitorOCISpec(spec *specs.Spec, monRes monitorResources, rootfsParams types.RootfsParams) *specs.Spec { + monSpec := *spec + monSpec.Root = &specs.Root{Path: rootfsParams.MonRootfs, Readonly: false} + monSpec.Mounts = monRes.Mounts + + process := *spec.Process + process.Cwd = "/" + monSpec.Process = &process + + linux := *spec.Linux + linux.Devices = monRes.Devices + linux.Namespaces = spec.Linux.Namespaces + monSpec.Linux = &linux + + return &monSpec +} + +// MonitorCapabilities returns the capability set the monitor process requires +// to finalize the execution environment for the monitor. Currently, it consists +// of the container's own set plus CAP_NET_ADMIN in all five sets. +// TODO: Narrow down to the extremely necessary capabilities. +func MonitorCapabilities(spec *specs.Spec) *configs.Capabilities { + caps := specCapabilities(spec) + if caps == nil { + uniklog.Warn("The container declares no capabilities. The monitor inherits urunc's own set.") + return nil + } + + for _, capab := range monCaps { + caps.Bounding = withCap(caps.Bounding, capab) + caps.Effective = withCap(caps.Effective, capab) + caps.Permitted = withCap(caps.Permitted, capab) + caps.Inheritable = withCap(caps.Inheritable, capab) + caps.Ambient = withCap(caps.Ambient, capab) + } + + return caps +} + +// specCapabilities copies the container's capability set out of the OCI spec, +// returning nil when the spec declares none. +func specCapabilities(spec *specs.Spec) *configs.Capabilities { + if spec.Process == nil || spec.Process.Capabilities == nil { + return nil + } + + caps := spec.Process.Capabilities + + return &configs.Capabilities{ + Bounding: slices.Clone(caps.Bounding), + Effective: slices.Clone(caps.Effective), + Permitted: slices.Clone(caps.Permitted), + Inheritable: slices.Clone(caps.Inheritable), + Ambient: slices.Clone(caps.Ambient), + } +} + +// withCap returns caps with add appended, unless it is already there. +func withCap(caps []string, add string) []string { + if slices.Contains(caps, add) { + return caps + } + + return append(caps, add) +} diff --git a/pkg/unikontainers/libcontainer_cgo.go b/pkg/unikontainers/libcontainer_cgo.go new file mode 100644 index 000000000..65bcb0b8e --- /dev/null +++ b/pkg/unikontainers/libcontainer_cgo.go @@ -0,0 +1,105 @@ +//go:build cgo + +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file holds the parts of the libcontainer monitor path that import runc's +// specconv and libcontainer packages, both of which require cgo. urunc is always +// built with cgo, so it always has them. The containerd shim is built without +// cgo and never uses the libcontainer runtime path, so gating these behind the +// cgo build tag keeps the shim free of the cgo-only dependency. See +// libcontainer_nocgo.go for the non-cgo stub that lets Delete still link. + +package unikontainers + +import ( + "fmt" + "path/filepath" + + "github.com/opencontainers/cgroups" + // Registers the cgroup device-rule setter (DevicesSetV1/V2). Without this + // blank import the setter stays nil and starting a container with device + // rules fails with "cgroup manager is not configured to set device rules". + _ "github.com/opencontainers/cgroups/devices" + devices "github.com/opencontainers/cgroups/devices/config" + "github.com/opencontainers/runc/libcontainer" + "github.com/opencontainers/runc/libcontainer/configs" + "github.com/opencontainers/runc/libcontainer/specconv" +) + +// BuildContainerConfig modifies the container's OCI spec, according to the monitor +// mounts and devices gathered during InitialSetup +func (u *Unikontainer) BuildContainerConfig(systemdCgroup bool) (*configs.Config, error) { + monRes, err := loadMonitorResources(u.BaseDir) + if err != nil { + return nil, fmt.Errorf("failed to load monitor resources: %w", err) + } + + monSpec := monitorOCISpec(u.Spec, monRes, monRes.Rootfs) + + config, err := specconv.CreateLibcontainerConfig(&specconv.CreateOpts{ + CgroupName: u.State.ID, + UseSystemdCgroup: systemdCgroup, + Spec: monSpec, + }) + if err != nil { + return nil, fmt.Errorf("failed to build the monitor's libcontainer config: %w", err) + } + + // urunc applies monitor-specific seccomp filters, therefore libcontainer + // should not apply any. + config.Seccomp = nil + + // Allow all devices through cgroups to avoid issues with device access from + // "urunc init" or the monitor process. + // TODO: Revisit this in the future and apply a stricter model. + config.Cgroups.Resources = &cgroups.Resources{ + Devices: []*devices.Rule{{Type: devices.WildcardDevice, Allow: true}}, + } + + // The Prestart, CreateRuntime, CreateContainer and StartContainer + // hooks are part of the configuration and are libcontainer's to run, + // Poststart and Poststop stay with urunc, in start and delete + // respectively, due to the extra steps urunc performs (sandbox and + // network cleanup). + delete(config.Hooks, configs.Poststart) + delete(config.Hooks, configs.Poststop) + + return config, nil +} + +// LibcontainerRoot returns the state directory libcontainer uses for the +// monitor's process execution environment. It is placed under the root directory +func (u *Unikontainer) LibcontainerRoot() string { + return filepath.Join(u.RootDir, libcontainerDirName) +} + +// destroyLibcontainer removes the monitor's libcontainer state directory and its +// cgroup. Following runc's delete. +func (u *Unikontainer) destroyLibcontainer() error { + container, err := libcontainer.Load(u.LibcontainerRoot(), u.State.ID) + if err != nil { + // Nothing to destroy: e.g. a create that failed before + // libcontainer.Create, or an already-completed teardown. + uniklog.Debugf("no monitor libcontainer state to destroy: %v", err) + return nil + } + + err = container.Destroy() + if err != nil { + return fmt.Errorf("failed to destroy the monitor's libcontainer state: %w", err) + } + + return nil +} diff --git a/pkg/unikontainers/libcontainer_nocgo.go b/pkg/unikontainers/libcontainer_nocgo.go new file mode 100644 index 000000000..daba9b773 --- /dev/null +++ b/pkg/unikontainers/libcontainer_nocgo.go @@ -0,0 +1,29 @@ +//go:build !cgo + +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The libcontainer monitor path requires cgo (runc's specconv and libcontainer +// packages). In a non-cgo build -- the containerd shim -- that path is never +// taken, so this stub stands in for destroyLibcontainer, which Delete references +// unconditionally. BuildContainerConfig and LibcontainerRoot need no stub: +// nothing compiled in a non-cgo build references them. + +package unikontainers + +// destroyLibcontainer is a no-op in a non-cgo build; see libcontainer_cgo.go for +// the real implementation. +func (u *Unikontainer) destroyLibcontainer() error { + return nil +} diff --git a/pkg/unikontainers/libcontainer_test.go b/pkg/unikontainers/libcontainer_test.go new file mode 100644 index 000000000..1187e7304 --- /dev/null +++ b/pkg/unikontainers/libcontainer_test.go @@ -0,0 +1,301 @@ +//go:build cgo + +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unikontainers + +import ( + "path/filepath" + "testing" + + devices "github.com/opencontainers/cgroups/devices/config" + "github.com/opencontainers/runc/libcontainer/configs" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// newTestUnikontainer builds the minimum Unikontainer that BuildContainerConfig +// needs: a spec and a monitor resources file. It returns the chosen rootfs +// params so the caller can pass them to BuildContainerConfig. The monitor rootfs +// is a real directory, because specconv resolves it and libcontainer's +// validation insists on an existing absolute path. +func newTestUnikontainer(t *testing.T, spec *specs.Spec, monRes monitorResources) (*Unikontainer, types.RootfsParams) { + t.Helper() + + baseDir := t.TempDir() + monRootfs := t.TempDir() + + rootfsParams := newRootfsResult("initrd", "initrd.cpio", "", monRootfs) + monRes.Rootfs = rootfsParams + + err := saveMonitorResources(baseDir, monRes) + require.NoError(t, err) + + u := &Unikontainer{ + BaseDir: baseDir, + RootDir: filepath.Dir(baseDir), + Spec: spec, + State: &specs.State{ + ID: "test-container", + }, + UruncCfg: defaultUruncConfig(), + } + + return u, rootfsParams +} + +// testSpec returns a spec close to what a real container carries: a couple of +// namespaces, a capability set and a cgroup path. +func testSpec() *specs.Spec { + return &specs.Spec{ + Version: specs.Version, + Root: &specs.Root{Path: "rootfs", Readonly: true}, + Process: &specs.Process{ + Cwd: "/guest/workdir", + Args: []string{"/unikernel"}, + Capabilities: &specs.LinuxCapabilities{ + Bounding: []string{"CAP_CHOWN", "CAP_KILL"}, + Effective: []string{"CAP_CHOWN"}, + Permitted: []string{"CAP_CHOWN", "CAP_KILL"}, + Inheritable: []string{"CAP_CHOWN"}, + Ambient: []string{"CAP_CHOWN"}, + }, + }, + Linux: &specs.Linux{ + CgroupsPath: "/pods/test-container", + Namespaces: []specs.LinuxNamespace{ + {Type: specs.NetworkNamespace, Path: "/proc/1234/ns/net"}, + {Type: specs.MountNamespace}, + {Type: specs.UTSNamespace}, + }, + }, + } +} + +// countString returns how many times want appears in values. +func countString(values []string, want string) int { + count := 0 + for _, v := range values { + if v == want { + count++ + } + } + + return count +} + +func TestBuildContainerConfig(t *testing.T) { + t.Run("roots the monitor at the monitor rootfs", func(t *testing.T) { + t.Parallel() + spec := testSpec() + u, rootfsParams := newTestUnikontainer(t, spec, monitorResources{}) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + assert.Equal(t, rootfsParams.MonRootfs, config.Rootfs) + // The container's rootfs is read-only, the monitor's is not. + assert.False(t, config.Readonlyfs) + assert.False(t, config.NoPivotRoot) + // The container's spec must not have been modified. + assert.Equal(t, "rootfs", spec.Root.Path) + assert.True(t, spec.Root.Readonly) + assert.Equal(t, "/guest/workdir", spec.Process.Cwd) + }) + + t.Run("maps the namespaces from the spec", func(t *testing.T) { + t.Parallel() + u, _ := newTestUnikontainer(t, testSpec(), monitorResources{}) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + assert.True(t, config.Namespaces.Contains(configs.NEWNET)) + assert.True(t, config.Namespaces.Contains(configs.NEWNS)) + assert.True(t, config.Namespaces.Contains(configs.NEWUTS)) + assert.False(t, config.Namespaces.Contains(configs.NEWPID)) + assert.False(t, config.Namespaces.Contains(configs.NEWUSER)) + assert.Equal(t, "/proc/1234/ns/net", config.Namespaces.PathOf(configs.NEWNET)) + }) + + t.Run("carries the monitor mounts and devices", func(t *testing.T) { + t.Parallel() + monRes := monitorResources{ + Mounts: []specs.Mount{ + tmpfsMount("/tmp", "65536k"), + bindMount("/usr/bin/qemu-system-x86_64", "/usr/bin/qemu-system-x86_64", true), + }, + Devices: []specs.LinuxDevice{ + {Path: "/dev/kvm", Type: "c", Major: 10, Minor: 232}, + {Path: "/dev/null", Type: "c", Major: 1, Minor: 3}, + }, + } + u, _ := newTestUnikontainer(t, testSpec(), monRes) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + require.Len(t, config.Mounts, 2) + assert.Equal(t, "/tmp", config.Mounts[0].Destination) + assert.Equal(t, "/usr/bin/qemu-system-x86_64", config.Mounts[1].Destination) + + // libcontainer adds the standard device nodes on top of the ones urunc + // gathered, and drops its own entry for a device the spec already + // declares, so /dev/null must appear exactly once. + paths := make([]string, 0, len(config.Devices)) + for _, dev := range config.Devices { + paths = append(paths, dev.Path) + } + assert.Contains(t, paths, "/dev/kvm") + assert.Contains(t, paths, "/dev/zero") + assert.Equal(t, 1, countString(paths, "/dev/null")) + }) + + t.Run("places the monitor in the container cgroup without limits", func(t *testing.T) { + t.Parallel() + spec := testSpec() + limit := int64(64 * 1024 * 1024) + spec.Linux.Resources = &specs.LinuxResources{ + Memory: &specs.LinuxMemory{Limit: &limit}, + } + u, _ := newTestUnikontainer(t, spec, monitorResources{}) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + assert.Equal(t, "/pods/test-container", config.Cgroups.Path) + assert.False(t, config.Cgroups.Systemd) + + // Only the allow-all device rule, and in particular no memory limit: + // the guest's RAM is sized from the same limit, so applying it to the + // monitor's cgroup would OOM-kill the VMM. + require.NotNil(t, config.Cgroups.Resources) + assert.Zero(t, config.Cgroups.Resources.Memory) + assert.Zero(t, config.Cgroups.Resources.CpuQuota) + require.Len(t, config.Cgroups.Resources.Devices, 1) + assert.Equal(t, devices.WildcardDevice, config.Cgroups.Resources.Devices[0].Type) + assert.True(t, config.Cgroups.Resources.Devices[0].Allow) + }) + + t.Run("honors the systemd cgroup driver", func(t *testing.T) { + t.Parallel() + spec := testSpec() + spec.Linux.CgroupsPath = "system.slice:urunc:test-container" + u, _ := newTestUnikontainer(t, spec, monitorResources{}) + + config, err := u.BuildContainerConfig(true) + require.NoError(t, err) + + assert.True(t, config.Cgroups.Systemd) + assert.Equal(t, "system.slice", config.Cgroups.Parent) + assert.Equal(t, "urunc", config.Cgroups.ScopePrefix) + assert.Equal(t, "test-container", config.Cgroups.Name) + }) + + t.Run("does not register the poststart and poststop hooks", func(t *testing.T) { + t.Parallel() + spec := testSpec() + hook := specs.Hook{Path: "/bin/true"} + spec.Hooks = &specs.Hooks{ + Prestart: []specs.Hook{hook}, + CreateRuntime: []specs.Hook{hook}, + CreateContainer: []specs.Hook{hook}, + StartContainer: []specs.Hook{hook}, + Poststart: []specs.Hook{hook}, + Poststop: []specs.Hook{hook}, + } + u, _ := newTestUnikontainer(t, spec, monitorResources{}) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + // urunc keeps its own timing for these two, since libcontainer would run + // Poststart from container.Start(), i.e. during create. + assert.False(t, config.HasHook(configs.Poststart)) + assert.False(t, config.HasHook(configs.Poststop)) + // The rest are libcontainer's to run. + assert.True(t, config.HasHook(configs.Prestart)) + assert.True(t, config.HasHook(configs.CreateRuntime)) + assert.True(t, config.HasHook(configs.CreateContainer)) + assert.True(t, config.HasHook(configs.StartContainer)) + }) + + // urunc delegates syscall filtering to the monitor and is never built with + // the seccomp build tag, so a non-nil Seccomp here would fail libcontainer's + // init. This guards against a specconv bump re-introducing it. + t.Run("never carries a seccomp profile", func(t *testing.T) { + t.Parallel() + spec := testSpec() + spec.Linux.Seccomp = &specs.LinuxSeccomp{ + DefaultAction: specs.ActErrno, + Architectures: []specs.Arch{specs.ArchX86_64}, + Syscalls: []specs.LinuxSyscall{ + {Names: []string{"read", "write"}, Action: specs.ActAllow}, + }, + } + u, _ := newTestUnikontainer(t, spec, monitorResources{}) + + config, err := u.BuildContainerConfig(false) + require.NoError(t, err) + + assert.Nil(t, config.Seccomp) + }) +} + +func TestMonitorCapabilities(t *testing.T) { + t.Run("adds CAP_NET_ADMIN to all five sets", func(t *testing.T) { + t.Parallel() + spec := testSpec() + + caps := MonitorCapabilities(spec) + + require.NotNil(t, caps) + for name, set := range map[string][]string{ + "bounding": caps.Bounding, + "effective": caps.Effective, + "permitted": caps.Permitted, + "inheritable": caps.Inheritable, + "ambient": caps.Ambient, + } { + assert.Contains(t, set, monCaps[0], "%s must carry CAP_NET_ADMIN", name) + assert.Contains(t, set, "CAP_CHOWN", "%s must keep the container's own capabilities", name) + } + + // The container's spec must not have been touched. + assert.NotContains(t, spec.Process.Capabilities.Bounding, monCaps[0]) + }) + + t.Run("does not duplicate a CAP_NET_ADMIN already present", func(t *testing.T) { + t.Parallel() + spec := testSpec() + spec.Process.Capabilities.Bounding = []string{monCaps[0]} + + caps := MonitorCapabilities(spec) + + require.NotNil(t, caps) + assert.Equal(t, []string{monCaps[0]}, caps.Bounding) + }) + + t.Run("a spec without capabilities gives nil", func(t *testing.T) { + t.Parallel() + spec := testSpec() + spec.Process.Capabilities = nil + + assert.Nil(t, MonitorCapabilities(spec)) + }) +} diff --git a/pkg/unikontainers/monitor.go b/pkg/unikontainers/monitor.go new file mode 100644 index 000000000..d282d5af5 --- /dev/null +++ b/pkg/unikontainers/monitor.go @@ -0,0 +1,125 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unikontainers + +import ( + "errors" + "fmt" + "os" + "syscall" + + m "github.com/urunc-dev/urunc/internal/metrics" + "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" + "github.com/urunc-dev/urunc/pkg/unikontainers/unikernels" +) + +// ExecMonitor is the entry point of the urunc process libcontainer starts +// inside the monitor's container. It reads the monitor spec, finalizes the +// environment for the monitor execution and execs it. +func ExecMonitor(metrics m.Writer) error { + // The ReadyPipeFD is passed as ExtraFle with libcontainer, but in order to do + // so, it gets CLOEXEC cleared. Therefore, restore it before spawning any child + // (e.g. virtiofsd from PreStartCmd): so the child does not inherit it. + syscall.CloseOnExec(ReadyPipeFD) + // this function is supposed to be called after the libcontainer has created the + // monitor container and is therefore in "/". + const pivotedRoot = "/" + + ms, err := LoadMonitorSpec(pivotedRoot) + if err != nil { + sigErr := signalReady(false) + if sigErr != nil { + uniklog.WithError(sigErr).Error("failed to signal the failed start") + } + return fmt.Errorf("failed to read the monitor spec: %w", err) + } + metrics.Capture(m.TS14) + + err = runMonitor(metrics, ms) + if err != nil { + uniklog.WithError(err).Error("setting up execution environment for monitor") + sigErr := signalReady(false) + if sigErr != nil { + uniklog.WithError(sigErr).Error("failed to signal the failed start") + } + + return errors.Join(err, sigErr) + } + + return nil +} + +// runMonitor sets up the monitor's network and guest command, drops to the +// container user and execve's the monitor. Every failure returns to caller +// On success it does not return. +func runMonitor(metrics m.Writer, ms MonitorSpec) error { + // The monitor's environment is not carried in the spec: libcontainer already + // gave this process the environment it configured for the monitor. + ms.ExecArgs.Environment = os.Environ() + + unikernel, err := unikernels.New(ms.UnikernelType) + if err != nil { + return err + } + + // TODO simplify this + monitorsConfig := map[string]types.MonitorConfig{ms.MonitorType: ms.MonitorCfg} + vmm, err := hypervisors.NewVMM(hypervisors.VmmType(ms.MonitorType), monitorsConfig) + if err != nil { + return err + } + + netArgs, err := SetupNet(ms.NetworkType, ms.User.UID, ms.User.GID) + if err != nil { + return fmt.Errorf("failed to setup network: %w", err) + } + metrics.Capture(m.TS16) + ms.ExecArgs.Net = netArgs + ms.GuestParams.Net = netArgs + + ms.ExecArgs.Command, err = buildUnikernelCommand(unikernel, ms.GuestParams) + if err != nil { + return err + } + + // Drop to the container user, which also clears the capabilities the tap + // device needed. From here on the monitor runs unprivileged. + err = setupUser(ms.User) + if err != nil { + return err + } + metrics.Capture(m.TS17) + + err = spawnProcess(ms.PreStartCmd) + if err != nil { + return err + } + + execCmd, err := vmm.BuildExecCmd(ms.ExecArgs, unikernel) + if err != nil { + return err + } + + // Report a successful setup to "urunc start" over the ready pipe, only after + // the command has been built. signalReady closes the pipe, so it is not left + // open in the monitor after the execve. + err = signalReady(true) + if err != nil { + return err + } + + return execMonitor(metrics, vmm, ms.ExecArgs, execCmd) +} diff --git a/pkg/unikontainers/monitor_spec.go b/pkg/unikontainers/monitor_spec.go new file mode 100644 index 000000000..85907fe0f --- /dev/null +++ b/pkg/unikontainers/monitor_spec.go @@ -0,0 +1,106 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unikontainers + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// monitorSpecFilename is the file that contains the information to finalize the +// setup of the monitor's execution environment. It is written into the monitor rootfs +// in order to let the process that libcontainer spawns to read it. +const monitorSpecFilename = ".monitor_spec.json" + +// MonitorSpec is everything the urunc monitor process needs in order to +// finalize the execution environment for the monitor process and then execve it. +type MonitorSpec struct { + ContainerID string `json:"containerID"` + UnikernelType string `json:"unikernelType"` + MonitorType string `json:"monitorType"` + MonitorCfg types.MonitorConfig `json:"monitorCfg"` + ExecArgs types.ExecArgs `json:"execArgs"` + GuestParams types.UnikernelParams `json:"guestParams"` + NetworkType string `json:"networkType"` + User specs.User `json:"user"` + PreStartCmd []string `json:"preStartCmd,omitempty"` +} + +// writeMonitorSpec builds the monitor spec and writes it into the monitor rootfs. +func (u *Unikontainer) writeMonitorSpec(rootfsParams types.RootfsParams, monRes monitorResources) error { + mSpec := u.buildMonitorSpec(rootfsParams, monRes) + + mSpec.NetworkType = u.getNetworkType() + mSpec.User = u.Spec.Process.User + + // The post-pivot process sees the monitor rootfs as "/", so the guest rootfs + // path it is handed has to be relative to it. + mSpec.GuestParams.Rootfs.MonRootfs = "/" + + // The monitor's environment is not persisted: the urunc monitor process + // inherits it from libcontainer anyway, and writing the host's environment + // into a file inside the monitor rootfs is exposure with no upside. + mSpec.ExecArgs.Environment = nil + + data, err := json.Marshal(mSpec) + if err != nil { + return fmt.Errorf("could not encode the monitor spec: %w", err) + } + + path, err := securejoin.SecureJoin(rootfsParams.MonRootfs, monitorSpecFilename) + if err != nil { + return fmt.Errorf("could not resolve path for monitor spec: %w", err) + } + + err = os.WriteFile(path, data, 0o600) + if err != nil { + return fmt.Errorf("could not write the monitor spec: %w", err) + } + + return nil +} + +// LoadMonitorSpec reads the monitor spec-file from dir +func LoadMonitorSpec(dir string) (MonitorSpec, error) { + var ms MonitorSpec + + data, err := os.ReadFile(filepath.Join(dir, monitorSpecFilename)) + if err != nil { + return ms, err + } + + err = json.Unmarshal(data, &ms) + if err != nil { + return ms, fmt.Errorf("could not decode the monitor spec: %w", err) + } + + return ms, nil +} + +// RemoveMonitorSpec deletes the monitor spec file from dir. +func RemoveMonitorSpec(dir string) error { + path, err := securejoin.SecureJoin(dir, monitorSpecFilename) + if err != nil { + return fmt.Errorf("could not resolve path for monitor spec: %w", err) + } + return os.Remove(path) +} diff --git a/pkg/unikontainers/monitor_spec_test.go b/pkg/unikontainers/monitor_spec_test.go new file mode 100644 index 000000000..bb5113824 --- /dev/null +++ b/pkg/unikontainers/monitor_spec_test.go @@ -0,0 +1,198 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unikontainers + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// newSpecUnikontainer builds the minimum Unikontainer that writeMonitorSpec +// needs: a spec, a state with the unikernel annotations, and the default urunc +// config. The returned RootfsParams points MonRootfs at monRootfs so the spec +// file lands in a real, writable directory. +func newSpecUnikontainer(t *testing.T, monRootfs string) (*Unikontainer, types.RootfsParams) { + t.Helper() + + u := &Unikontainer{ + State: &specs.State{ + ID: "test-container", + Annotations: map[string]string{ + annotType: "unikraft", + annotHypervisor: "qemu", + annotVersion: "1.0", + annotBinary: "/unikernel", + }, + }, + Spec: &specs.Spec{ + Version: specs.Version, + Root: &specs.Root{Path: "rootfs", Readonly: true}, + Process: &specs.Process{ + Args: []string{"/unikernel", "--flag"}, + Env: []string{"HOME=/root"}, + Cwd: "/guest/workdir", + User: specs.User{UID: 1000, GID: 1000}, + }, + Linux: &specs.Linux{ + Seccomp: &specs.LinuxSeccomp{DefaultAction: specs.ActErrno}, + }, + Annotations: map[string]string{}, + }, + UruncCfg: defaultUruncConfig(), + } + + rootfsParams := types.RootfsParams{Type: "initrd", Path: "initrd.cpio", MonRootfs: monRootfs} + + return u, rootfsParams +} + +// readMonitorSpecFile decodes the spec file written into dir directly, so the +// write-side test does not depend on LoadMonitorSpec (the read side). +func readMonitorSpecFile(t *testing.T, dir string) MonitorSpec { + t.Helper() + + data, err := os.ReadFile(filepath.Join(dir, monitorSpecFilename)) + require.NoError(t, err) + + var ms MonitorSpec + err = json.Unmarshal(data, &ms) + require.NoError(t, err) + + return ms +} + +func TestWriteMonitorSpec(t *testing.T) { + t.Run("writes a spec that decodes back unchanged", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + got := readMonitorSpecFile(t, monRootfs) + + assert.Equal(t, "test-container", got.ContainerID) + assert.Equal(t, "unikraft", got.UnikernelType) + assert.Equal(t, "qemu", got.MonitorType) + assert.Equal(t, u.UruncCfg.Monitors["qemu"], got.MonitorCfg) + assert.Equal(t, specs.User{UID: 1000, GID: 1000}, got.User) + // No knative annotation, so the network type is dynamic. + assert.Equal(t, "dynamic", got.NetworkType) + // The post-pivot process sees the monitor rootfs as "/". + assert.Equal(t, "/", got.GuestParams.Rootfs.MonRootfs) + }) + + t.Run("does not persist the monitor environment", func(t *testing.T) { + // t.Setenv forbids t.Parallel. + t.Setenv("URUNC_TEST_SECRET", "do-not-write-me") + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + got := readMonitorSpecFile(t, monRootfs) + assert.Nil(t, got.ExecArgs.Environment) + + // Not just absent from the decoded struct: absent from the file. + data, err := os.ReadFile(filepath.Join(monRootfs, monitorSpecFilename)) + require.NoError(t, err) + assert.NotContains(t, string(data), "do-not-write-me") + }) + + t.Run("keeps seccomp enabled when the spec confines it", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + got := readMonitorSpecFile(t, monRootfs) + assert.True(t, got.ExecArgs.Seccomp) + }) + + t.Run("disables seccomp for an unconfined spec", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + u.Spec.Linux.Seccomp = nil + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + got := readMonitorSpecFile(t, monRootfs) + assert.False(t, got.ExecArgs.Seccomp) + }) + + t.Run("writes the file owner-only inside the monitor rootfs", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + info, err := os.Stat(filepath.Join(monRootfs, monitorSpecFilename)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + }) +} + +func TestMonitorSpecFile(t *testing.T) { + t.Run("can be loaded back after being written", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + got, err := LoadMonitorSpec(monRootfs) + require.NoError(t, err) + assert.Equal(t, "test-container", got.ContainerID) + }) + + t.Run("can be removed once it has been read", func(t *testing.T) { + t.Parallel() + monRootfs := t.TempDir() + u, rootfsParams := newSpecUnikontainer(t, monRootfs) + + err := u.writeMonitorSpec(rootfsParams, monitorResources{}) + require.NoError(t, err) + + err = RemoveMonitorSpec(monRootfs) + require.NoError(t, err) + + _, err = LoadMonitorSpec(monRootfs) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("reports a missing file", func(t *testing.T) { + t.Parallel() + + _, err := LoadMonitorSpec(t.TempDir()) + assert.ErrorIs(t, err, os.ErrNotExist) + }) +} diff --git a/pkg/unikontainers/rootfs.go b/pkg/unikontainers/rootfs.go index bd1c800f0..40dd5b963 100644 --- a/pkg/unikontainers/rootfs.go +++ b/pkg/unikontainers/rootfs.go @@ -30,17 +30,13 @@ import ( // TODO: Find and set the correct size for the tmpfs in the host const tmpfsSizeForNoRootfs = "65536k" -// annotRootfsParams holds JSON RootfsParams after shim chooseGuestRootfs. -// When present in bundle config.json, Exec reuses it; otherwise Exec runs ChooseRootfs. -const annotRootfsParams = "com.urunc.internal.rootfs.params" - type rootfsBuilder interface { preSetup() error postSetup() error getMounts() ([]specs.Mount, error) getBlockDevs() ([]types.BlockDevParams, error) getSharedDirs() (types.SharedfsParams, error) - preStart() error + preStartCmd() []string } // tmpfsMount creates a mount for a tmpfs in the form of "/tmp" at target @@ -162,7 +158,7 @@ func (n noRootfs) getSharedDirs() (types.SharedfsParams, error) { return types.SharedfsParams{}, nil } -func (n noRootfs) preStart() error { +func (n noRootfs) preStartCmd() []string { return nil } diff --git a/pkg/unikontainers/shared_fs.go b/pkg/unikontainers/shared_fs.go index 9441f39a7..875d9dc6f 100644 --- a/pkg/unikontainers/shared_fs.go +++ b/pkg/unikontainers/shared_fs.go @@ -15,7 +15,6 @@ package unikontainers import ( - "fmt" "path/filepath" "strings" @@ -73,26 +72,24 @@ func (s sharedfsRootfs) getSharedDirs() (types.SharedfsParams, error) { }, nil } -func (s sharedfsRootfs) preStart() error { +func (s sharedfsRootfs) preStartCmd() []string { if s.sfsType == "9pfs" { return nil } - // Start the virtiofsd process - args := []string{ + // The virtiofsd argv, with the binary itself as the first element so it can be + // both stored in the monitor spec and spawned later by spawnProcess. + argv := []string{ + s.vfsdConfig.Path, "--socket-path=/tmp/vhostqemu", "--shared-dir", s.sharedPath, } if s.vfsdConfig.Options != "" { - args = append(args, strings.Fields(s.vfsdConfig.Options)...) + argv = append(argv, strings.Fields(s.vfsdConfig.Options)...) } - err := spawnProcess(s.vfsdConfig.Path, args) - if err != nil { - err = fmt.Errorf("failed to start virtiofsd: %w", err) - } - return err + return argv } func chooseTmpfsSize(sfsType string, mem uint64) string { diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index ad0bf2897..5b0a367b2 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -44,6 +44,8 @@ import ( const ( monitorRootfsDirName string = "monRootfs" containerRootfsMountPath string = "/cntrRootfs" + // libcontainerDirName is the directory under urunc's root used from libcontainer + libcontainerDirName string = "libcontainer" ) var uniklog = logrus.WithField("subsystem", "unikontainers") @@ -61,6 +63,9 @@ type Unikontainer struct { UruncCfg *UruncConfig Listener *net.UnixListener Conn *net.UnixConn + // readyPipe is the read end of the libcontainer-mode ready FIFO, held by + // "urunc start" between opening it and reading the monitor's outcome. + readyPipe *os.File } // New parses the bundle and creates a new Unikontainer object @@ -90,6 +95,8 @@ func New(bundlePath string, containerID string, rootDir string, cfg *UruncConfig return nil, ErrNotUnikernel } + uniklog.Debugf("libcontainer runtime enabled: %t", cfg.Runtime.Libcontainer) + confMap := config.Map() maps.Copy(confMap, cfg.Map()) @@ -136,6 +143,7 @@ func Get(containerID string, rootDir string) (*Unikontainer, error) { u.RootDir = rootDir u.Spec = spec u.UruncCfg = UruncConfigFromMap(state.Annotations) + uniklog.Debugf("libcontainer runtime enabled: %t", u.UruncCfg.Runtime.Libcontainer) return u, nil } @@ -169,26 +177,10 @@ func (u *Unikontainer) InitialSetup() error { // if the respective annotation is set then, depending on the guest // (supports block or 9pfs), it will use the supported option. In case // both ae supported, then the block option will be used by default. - var rootfsParams types.RootfsParams - - // Read the rootfs choice written by the shim. - if rootfsParamsJSON := u.Spec.Annotations[annotRootfsParams]; rootfsParamsJSON != "" { - if err := json.Unmarshal([]byte(rootfsParamsJSON), &rootfsParams); err != nil { - return fmt.Errorf("could not decode guest rootfs params: %w", err) - } - } - - if rootfsParams.MonRootfs == "" { - rootfsParams, err = ChooseRootfs(bundleDir, rootfsDir, u.State.Annotations, u.UruncCfg) - if err != nil { - uniklog.Errorf("could not choose guest rootfs: %v", err) - return err - } - encoded, err := json.Marshal(rootfsParams) - if err != nil { - return err - } - u.State.Annotations[annotRootfsParams] = string(encoded) + rootfsParams, err := ChooseRootfs(bundleDir, rootfsDir, u.State.Annotations, u.UruncCfg) + if err != nil { + uniklog.Errorf("could not choose guest rootfs: %v", err) + return err } uniklog.WithFields(logrus.Fields{ "rootfs_type": rootfsParams.Type, @@ -216,6 +208,12 @@ func (u *Unikontainer) InitialSetup() error { if err != nil { return err } + monRes.Rootfs = rootfsParams + + err = rfsBuilder.postSetup() + if err != nil { + return fmt.Errorf("post setup step for rootfs failed: %w", err) + } u.State.Status = specs.StateCreating // FIXME: should we really create this base dir @@ -229,6 +227,17 @@ func (u *Unikontainer) InitialSetup() error { return fmt.Errorf("failed to store monitor resources: %w", err) } + // In libcontainer mode the monitor process runs inside a dedicated rootfs and + // cannot reach the state directory, so everything it needs is written into that + // rootfs now. + // TODO: Switch to fifo + if u.UruncCfg.Runtime.Libcontainer { + err = u.writeMonitorSpec(rootfsParams, monRes) + if err != nil { + return fmt.Errorf("failed to store the monitor spec: %w", err) + } + } + return u.saveContainerState() } @@ -255,8 +264,9 @@ func (u *Unikontainer) SetRunningState() error { return u.saveContainerState() } -func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { - networkType := u.getNetworkType() +// SetupNet creates the sandbox's network device (tap) in the current network +// namespace and returns its parameters; uid and gid own the tap device. +func SetupNet(networkType string, uid, gid uint32) (types.NetDevParams, error) { uniklog.WithField("network type", networkType).Debug("Retrieved network type") netArgs := types.NetDevParams{} netManager, err := network.NewNetworkManager(networkType) @@ -264,7 +274,7 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { return netArgs, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) } - networkInfo, err := netManager.NetworkSetup(u.Spec.Process.User.UID, u.Spec.Process.User.GID) + networkInfo, err := netManager.NetworkSetup(uid, gid) if err != nil { // TODO: Handle this case better. We do not need to show an error // since there was no network in the container. Therefore, we @@ -388,6 +398,13 @@ func getMonitorResources(rfs rootfsBuilder, rootfsParams types.RootfsParams, vmm } res.Devices = append(res.Devices, blockDevs...) + res.Sharedfs, err = rfs.getSharedDirs() + if err != nil { + return res, fmt.Errorf("failed to get directories to share with sandbox: %w", err) + } + + res.PreStartCmd = rfs.preStartCmd() + return res, nil } @@ -446,45 +463,16 @@ func monitorMemoryBytes(defaultMem uint, resources *specs.LinuxResources) uint64 return mem } -// nolint:gocyclo -func (u *Unikontainer) Exec(metrics m.Writer) error { - metrics.Capture(m.TS15) +func (u *Unikontainer) buildMonitorSpec(rootfsParams types.RootfsParams, monRes monitorResources) MonitorSpec { + var mSpec MonitorSpec - // container Paths - // Make sure paths are clean - bundleDir := filepath.Clean(u.State.Bundle) - rootfsDir := filepath.Clean(u.Spec.Root.Path) - rootfsDir, err := resolveAgainstBase(bundleDir, rootfsDir) - if err != nil { - uniklog.Errorf("could not resolve rootfs directory %s: %v", rootfsDir, err) - return err - } - - // unikernel unikernelType := u.State.Annotations[annotType] - unikernel, err := unikernels.New(unikernelType) - if err != nil { - return err - } - - // Vmm vmmType := u.State.Annotations[annotHypervisor] - vmm, err := hypervisors.NewVMM(hypervisors.VmmType(vmmType), u.UruncCfg.Monitors) - if err != nil { - return err - } - - // unikernelParams unikernelVersion := u.State.Annotations[annotVersion] - - // ExecArgs unikernelPath := u.State.Annotations[annotBinary] initrdPath := u.State.Annotations[annotInitrd] - // debug uniklog.WithFields(logrus.Fields{ - "bundle directory": bundleDir, - "rootfs directory": rootfsDir, "vmm type": vmmType, "unikernel type": unikernelType, "unikernel version": unikernelVersion, @@ -492,14 +480,12 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { "initrd Path": initrdPath, }).Debug("Initialization values") - // ExecArgs defaultVCPUs := u.UruncCfg.Monitors[vmmType].DefaultVCPUs if defaultVCPUs < 1 { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB - // ExecArgs vmmArgs := types.ExecArgs{ ContainerID: u.State.ID, UnikernelPath: unikernelPath, @@ -510,127 +496,140 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { Environment: os.Environ(), } - // ExecArgs // Check if container is set to unconfined -- disable seccomp if u.Spec.Linux.Seccomp == nil { uniklog.Warn("Seccomp is disabled") vmmArgs.Seccomp = false } - procAttrs := types.ProcessConfig{ - UID: u.Spec.Process.User.UID, - GID: u.Spec.Process.User.GID, - WorkDir: u.Spec.Process.Cwd, - } - // UnikernelParams - // populate unikernel params - unikernelParams := types.UnikernelParams{ - CmdLine: u.Spec.Process.Args, - EnvVars: u.Spec.Process.Env, - Monitor: vmmType, - Version: unikernelVersion, - ProcConf: procAttrs, + guest := types.UnikernelParams{ + CmdLine: u.Spec.Process.Args, + EnvVars: u.Spec.Process.Env, + Monitor: vmmType, + Version: unikernelVersion, + ProcConf: types.ProcessConfig{ + UID: u.Spec.Process.User.UID, + GID: u.Spec.Process.User.GID, + WorkDir: u.Spec.Process.Cwd, + }, NetDevName: u.State.Annotations[annotNetDev], BlkDevName: u.State.Annotations[annotBlkDev], + Rootfs: rootfsParams, + Block: monRes.BlockArgs, } - if len(unikernelParams.CmdLine) == 0 { - unikernelParams.CmdLine = strings.Fields(u.State.Annotations[annotCmdLine]) + if len(guest.CmdLine) == 0 { + guest.CmdLine = strings.Fields(u.State.Annotations[annotCmdLine]) } - // handle network - netArgs, err := u.SetupNet() - if err != nil { - uniklog.Errorf("failed to setup network: %v", err) - return err + if rootfsParams.Type == "virtiofs" || rootfsParams.Type == "9pfs" { + // Update the paths of the files we need to pass in the monitor process. + vmmArgs.UnikernelPath = adjustPathsForSharedfs(vmmArgs.UnikernelPath) + vmmArgs.InitrdPath = adjustPathsForSharedfs(vmmArgs.InitrdPath) } - metrics.Capture(m.TS16) - withTUNTAP := netArgs.IP != "" - - // UnikernelParams - unikernelParams.Net = netArgs + vmmArgs.Sharedfs = monRes.Sharedfs - // ExecArgs - vmmArgs.Net = netArgs + mSpec.ContainerID = u.State.ID + mSpec.UnikernelType = unikernelType + mSpec.MonitorType = vmmType + mSpec.MonitorCfg = u.UruncCfg.Monitors[vmmType] + mSpec.ExecArgs = vmmArgs + mSpec.GuestParams = guest + mSpec.PreStartCmd = monRes.PreStartCmd - // guest rootfs - // block - // handle guest's rootfs. - // There are three options: - // 1. No rootfs for guest - // 2. Use the devmapper snapshot as a block device for the guest's rootfs - // 3. Use 9pfs to share the container's rootfs as the guest's rootfs - // By default, urunc will not set any rootfs for the guest. However, - // if the respective annotation is set then, depending on the guest - // (supports block or 9pfs), it will use the supported option. In case - // both ae supported, then the block option will be used by default. - var rootfsParams types.RootfsParams + return mSpec +} - // Read the rootfs choice written by the shim. - if rootfsParamsJSON := u.State.Annotations[annotRootfsParams]; rootfsParamsJSON != "" { - if err := json.Unmarshal([]byte(rootfsParamsJSON), &rootfsParams); err != nil { - return fmt.Errorf("could not decode guest rootfs params: %w", err) - } +// setupMonitorRootfs prepares the monitor rootfs: it makes sure the directory +// exists and is mounted with a propagation flag that allows a later pivot, then +// replicates the gathered mounts and devices inside it and gives the monitor a +// console. +func (u *Unikontainer) setupMonitorRootfs(monRootfs string, monRes monitorResources, withTUNTAP bool) error { + err := os.MkdirAll(monRootfs, 0o755) + if err != nil { + return fmt.Errorf("failed to create monitor rootfs directory %s: %w", monRootfs, err) } - if rootfsParams.MonRootfs == "" { - uniklog.Errorf("missing annotations from selected rootfs") - return fmt.Errorf("missing metadata for rootfs preparation") + // Make sure that rootfs is mounted with the correct propagation + // flags so we can later pivot if needed. + err = prepareRoot(monRootfs, u.Spec.Linux.RootfsPropagation) + if err != nil { + return err } - uniklog.WithFields(logrus.Fields{ - "rootfs_type": rootfsParams.Type, - "rootfs_path": rootfsParams.Path, - "mon_rootfs": rootfsParams.MonRootfs, - }).Debug("guest rootfs params") - rfsBuilder := u.newRootfsBuilder(rootfsParams, unikernel, unikernelPath, initrdPath, vmmArgs.MemSizeB) - if rootfsParams.Type == "virtiofs" || rootfsParams.Type == "9pfs" { - // Update the paths of the files we need to pass in the monitor process. - vmmArgs.UnikernelPath = adjustPathsForSharedfs(vmmArgs.UnikernelPath) - vmmArgs.InitrdPath = adjustPathsForSharedfs(vmmArgs.InitrdPath) + err = applyMounts(monRootfs, monRes.Mounts) + if err != nil { + return fmt.Errorf("failed to apply rootfs mounts: %w", err) } - if err = os.MkdirAll(rootfsParams.MonRootfs, 0o755); err != nil { - return fmt.Errorf("failed to create monitor rootfs directory %s: %w", rootfsParams.MonRootfs, err) + // setupDevices decides whether to create the TUN/TAP device based on the + // container's network configuration. + err = setupDevices(monRootfs, monRes.Devices, withTUNTAP) + if err != nil { + return fmt.Errorf("failed to create devices in monitor rootfs: %w", err) } - // Prepare Monitor rootfs - // Make sure that rootfs is mounted with the correct propagation - // flags so we can later pivot if needed. - err = prepareRoot(rootfsParams.MonRootfs, u.Spec.Linux.RootfsPropagation) + err = setupConsole(monRootfs) if err != nil { - return err + return fmt.Errorf("failed to setup console: %w", err) } - // The monitor mounts and devices were gathered and stored in a file during - // InitialSetup; here we just apply them. postSetup applies the container's own - // bind mounts on top of the shared rootfs, and setupDevices decides whether to - // create the TUN/TAP device based on the container's network configuration. + return nil +} + +// nolint:gocyclo +func (u *Unikontainer) Exec(metrics m.Writer) error { + metrics.Capture(m.TS15) + + // The chosen guest rootfs params, together with the monitor mounts, devices + // and block args, were gathered and stored in monitor.json during + // InitialSetup. Load them back here. monRes, err := loadMonitorResources(u.BaseDir) if err != nil { return fmt.Errorf("failed to load monitor resources: %w", err) } + rootfsParams := monRes.Rootfs + if rootfsParams.MonRootfs == "" { + uniklog.Errorf("missing metadata for selected rootfs") + return fmt.Errorf("missing metadata for rootfs preparation") + } + uniklog.WithFields(logrus.Fields{ + "rootfs_type": rootfsParams.Type, + "rootfs_path": rootfsParams.Path, + "mon_rootfs": rootfsParams.MonRootfs, + }).Debug("guest rootfs params") + + ms := u.buildMonitorSpec(rootfsParams, monRes) + vmmArgs := ms.ExecArgs + unikernelParams := ms.GuestParams - err = applyMounts(rootfsParams.MonRootfs, monRes.Mounts) + // The spec carries the monitor and unikernel by type; rebuild the behaviour + // objects it refers to, exactly as the monitor process does. + unikernel, err := unikernels.New(ms.UnikernelType) if err != nil { - return fmt.Errorf("failed to apply rootfs mounts: %w", err) + return err } - - err = rfsBuilder.postSetup() + vmm, err := hypervisors.NewVMM(hypervisors.VmmType(ms.MonitorType), u.UruncCfg.Monitors) if err != nil { - return fmt.Errorf("post setup step for rootfs failed: %w", err) + return err } - err = setupDevices(rootfsParams.MonRootfs, monRes.Devices, withTUNTAP) + // handle network + netArgs, err := SetupNet(u.getNetworkType(), u.Spec.Process.User.UID, u.Spec.Process.User.GID) if err != nil { - return fmt.Errorf("failed to create devices in monitor rootfs: %w", err) + uniklog.Errorf("failed to setup network: %v", err) + return err } + metrics.Capture(m.TS16) + withTUNTAP := netArgs.IP != "" + unikernelParams.Net = netArgs + vmmArgs.Net = netArgs - err = setupConsole(rootfsParams.MonRootfs) + err = u.setupMonitorRootfs(rootfsParams.MonRootfs, monRes, withTUNTAP) if err != nil { return err } - metrics.Capture(m.TS17) + // vAccel setup vAccelType, vsockSocketPath, rpcAddress, err := resolveVAccelConfig(u.State.Annotations[annotHypervisor], u.Spec.Annotations) if err != nil { @@ -664,39 +663,13 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { vmmArgs.VSockDevID = idToGuestCID(u.State.ID) } - unikernelParams.Rootfs = rootfsParams - - // unikernelParams - // The block parameters were gathered in InitialSetup and stored in the - // monitor resources file. - unikernelParams.Block = monRes.BlockArgs - - // ExecArgs - sharedfsArgs, err := rfsBuilder.getSharedDirs() - if err != nil { - return fmt.Errorf("failed to get directories to share with sandbox: %w", err) - } - vmmArgs.Sharedfs = sharedfsArgs - - // unikernel - err = unikernel.Init(unikernelParams) - if errors.Is(err, unikernels.ErrUndefinedVersion) || - errors.Is(err, unikernels.ErrVersionParsing) { - uniklog.WithError(err).Error("an error occurred while initializing the unikernel") - } else if err != nil { - return err - } - // unikernel // build the unikernel command - unikernelCmd, err := unikernel.CommandString() + vmmArgs.Command, err = buildUnikernelCommand(unikernel, unikernelParams) if err != nil { return err } - // ExecArgs - vmmArgs.Command = unikernelCmd - // pivot _, err = findNS(u.Spec.Linux.Namespaces, specs.MountNamespace) // Only pivot if a mount namespace entry is actually present in the @@ -728,35 +701,54 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - err = rfsBuilder.preStart() + err = spawnProcess(monRes.PreStartCmd) if err != nil { return err } - uniklog.Debug("calling vmm execve") - metrics.Capture(m.TS18) - - // Build the VMM command once and verify it can be constructed successfully. - // This ensures we don't report the container as started if command building fails. + // Build the VMM command once and verify it can be constructed successfully, so + // we do not report the container as started if command building fails. execCmd, err := vmm.BuildExecCmd(vmmArgs, unikernel) if err != nil { uniklog.WithError(err).Error("failed to build VMM command") return err } - // Notify urunc start that the monitor is ready to execute. - // We send this after BuildExecCmd succeeds to avoid reporting a container - // as started when the VMM command cannot be built. - // TODO: The container can still be reported as running if the PreExec step - // (e.g., BPF/seccomp filter setup) fails after this point. We should find - // a way to handle that case as well. + // Notify urunc start that the monitor is ready to execute, only after the + // command builds so a container is never reported started when it cannot be. err = u.SendMessage(StartSuccess) if err != nil { return err } + return execMonitor(metrics, vmm, vmmArgs, execCmd) +} + +// buildUnikernelCommand initializes the unikernel with the collected parameters +// and returns its command line. +func buildUnikernelCommand(unikernel types.Unikernel, params types.UnikernelParams) (string, error) { + err := unikernel.Init(params) + if errors.Is(err, unikernels.ErrUndefinedVersion) || + errors.Is(err, unikernels.ErrVersionParsing) { + uniklog.WithError(err).Error("an error occurred while initializing the unikernel") + } else if err != nil { + return "", err + } + + return unikernel.CommandString() +} + +// execMonitor runs the monitor's pre-exec setup and finally execve's the monitor. +// It does not return on success: +// +// TODO: The container can still be reported as running if the PreExec step +// (e.g., BPF/seccomp filter setup) fails after the caller reported success. We +// should find a way to handle that case as well. +func execMonitor(metrics m.Writer, vmm types.VMM, execArgs types.ExecArgs, execCmd []string) error { + uniklog.Debug("calling vmm execve") + metrics.Capture(m.TS18) // Perform any monitor-specific pre-exec setup (e.g., seccomp filters for HVT). - err = vmm.PreExec(vmmArgs) + err := vmm.PreExec(execArgs) if err != nil { uniklog.WithError(err).Error("failed to perform pre-exec setup") return err @@ -764,7 +756,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // Execute the VMM using the command we built earlier. uniklog.WithField("command", execCmd).Debug("Ready to execve VMM") - return syscall.Exec(vmm.Path(), execCmd, vmmArgs.Environment) //nolint: gosec + return syscall.Exec(vmm.Path(), execCmd, execArgs.Environment) //nolint: gosec } func setupUser(user specs.User) error { @@ -848,6 +840,15 @@ func (u *Unikontainer) Delete() error { return fmt.Errorf("cannot delete running container: %s", u.State.ID) } + // In libcontainer mode, tear down the monitor's libcontainer state and cgroup. + // Like runc. + if u.UruncCfg.Runtime.Libcontainer { + err := u.destroyLibcontainer() + if err != nil { + return err + } + } + // Restore the block volume mounts that were unmounted during create, // so their sources become discoverable by future containers. Do it in // a best-effort way, since a failure to restore a mount should not @@ -891,11 +892,17 @@ func (u *Unikontainer) Delete() error { dirs = append(dirs, monitorRootfsDirName) prefPath = bundleDir } else { - // Otherwise remove the enw directories we created inside the - // container's rootfs. + // Otherwise remove the enw directories we created and the monitor spec + // file inside the container's rootfs. // We do not need to unmount anything here, since we rely on Linux // to do the cleanup for us. This will happen automatically, // when the mount namespace gets destroyed + err = RemoveMonitorSpec(rootfsDir) + // Ignore the case where the file does not exist. + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to remove the monitor spec: %w", err) + } + dirs = []string{ "/lib", "/lib64", @@ -1314,6 +1321,12 @@ func (u *Unikontainer) FormatNsenterInfo() (rdr io.Reader, retErr error) { // If it is not the reexec process then the listener will refer to the // uruncSock, the socket that holds messages from reexec to urunc instances func (u *Unikontainer) CreateListener(isReexec bool) error { + // In libcontainer mode the start side reads the monitor's outcome from a FIFO + // in the state dir, whose write end the monitor inherited from create. + if !isReexec && u.UruncCfg.Runtime.Libcontainer { + return u.openReadReadyPipe() + } + sockAddr := getUruncSockAddr(u.BaseDir) if isReexec { sockAddr = getReexecSockAddr(u.BaseDir) @@ -1332,6 +1345,10 @@ func (u *Unikontainer) CreateListener(isReexec bool) error { // DestroyListener destroys an existing listener over a socket func (u *Unikontainer) DestroyListener(isReexec bool) error { + if !isReexec && u.UruncCfg.Runtime.Libcontainer { + return u.closeReadReadyPipe() + } + sockAddr := getUruncSockAddr(u.BaseDir) if isReexec { sockAddr = getReexecSockAddr(u.BaseDir) @@ -1389,6 +1406,10 @@ func (u *Unikontainer) DestroyConn(isReexec bool) error { // AwaitMessage waits for a specific message in the listener of unikontainer instance func (u *Unikontainer) AwaitMsg(msg IPCMessage) error { + // In libcontainer mode the monitor reports over the ready FIFO. + if u.UruncCfg.Runtime.Libcontainer { + return u.awaitReadyPipe() + } return AwaitMessage(u.Listener, msg) } diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 22573f43c..b4f1a027b 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -34,13 +34,42 @@ type UruncTimestamps struct { Destination string `toml:"destination"` // Used to specify a file for timestamps } +// UruncRuntime holds runtime-wide behavior options. +type UruncRuntime struct { + // Libcontainer selects whether the monitor's execution environment + // is set up through runc's libcontainer instead of urunc's own implementation. + Libcontainer bool `toml:"libcontainer"` +} + +// libcontainerKey is the state.json annotation with the runtime setting +// for libcontainer. +const libcontainerKey = "urunc_config.runtime.libcontainer" + type UruncConfig struct { Log UruncLog `toml:"log"` Timestamps UruncTimestamps `toml:"timestamps"` + Runtime UruncRuntime `toml:"runtime"` Monitors map[string]types.MonitorConfig `toml:"monitors"` ExtraBins map[string]types.ExtraBinConfig `toml:"extra_binaries"` } +// runtimeFromMap rebuilds the runtime options from the state.json annotations. +// A missing or malformed value falls back to the default (libcontainer off). +func runtimeFromMap(cfgMap map[string]string) UruncRuntime { + rt := defaultRuntimeConfig() + val, ok := cfgMap[libcontainerKey] + if !ok { + return rt + } + choice, err := strconv.ParseBool(val) + if err != nil { + uniklog.Warnf("Invalid libcontainer value %q. Using default (false).", val) + return rt + } + rt.Libcontainer = choice + return rt +} + // this struct is used to parse only the log and timestamp section of the urunc config file type LogMetricsUruncConfig struct { Log UruncLog `toml:"log"` @@ -78,6 +107,12 @@ func defaultTimestampsConfig() UruncTimestamps { } } +func defaultRuntimeConfig() UruncRuntime { + return UruncRuntime{ + Libcontainer: false, + } +} + const ( defaultMonitorMemoryMB uint = 256 defaultMonitorVCPUs uint = 1 @@ -103,6 +138,7 @@ func defaultUruncConfig() *UruncConfig { return &UruncConfig{ Log: defaultLogConfig(), Timestamps: defaultTimestampsConfig(), + Runtime: defaultRuntimeConfig(), Monitors: defaultMonitorsConfig(), ExtraBins: defaultExtraBinConfig(), } @@ -138,6 +174,7 @@ func (p *UruncConfig) Map() map[string]string { // them to this map. this map will be used to save the rest of the urunc config to state.json cfgMap := make(map[string]string) + cfgMap[libcontainerKey] = strconv.FormatBool(p.Runtime.Libcontainer) for hv, hvCfg := range p.Monitors { prefix := "urunc_config.monitors." + hv + "." cfgMap[prefix+"default_memory_mb"] = strconv.FormatUint(uint64(hvCfg.DefaultMemoryMB), 10) @@ -158,6 +195,7 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { // since log and timestamps are loaded at the start of urunc, we will not be reading // them from this map. this map will be used to parse the rest of the urunc config from state.json cfg := &UruncConfig{ + Runtime: runtimeFromMap(cfgMap), Monitors: defaultMonitorsConfig(), ExtraBins: defaultExtraBinConfig(), } diff --git a/pkg/unikontainers/urunc_config_test.go b/pkg/unikontainers/urunc_config_test.go index a51a3eb96..a1ba139bf 100644 --- a/pkg/unikontainers/urunc_config_test.go +++ b/pkg/unikontainers/urunc_config_test.go @@ -25,6 +25,7 @@ import ( // Constants for test configuration keys and values const ( + testLibcontainerKey = "urunc_config.runtime.libcontainer" testQemuMemoryKey = "urunc_config.monitors.qemu.default_memory_mb" testQemuVCPUsKey = "urunc_config.monitors.qemu.default_vcpus" testQemuBinaryKey = "urunc_config.monitors.qemu.binary_path" @@ -425,7 +426,7 @@ func TestUruncConfigMap(t *testing.T) { assert.Equal(t, config.ExtraBins["custom"].Options, cfgMap["urunc_config.extra_binaries.custom.options"]) }) - t.Run("empty monitors map produces empty result", func(t *testing.T) { + t.Run("empty monitors map produces no monitor keys", func(t *testing.T) { t.Parallel() config := &UruncConfig{ Monitors: map[string]types.MonitorConfig{}, @@ -434,10 +435,10 @@ func TestUruncConfigMap(t *testing.T) { cfgMap := config.Map() assert.NotNil(t, cfgMap) - assert.Empty(t, cfgMap) + assert.Equal(t, map[string]string{testLibcontainerKey: "false"}, cfgMap) }) - t.Run("empty extra binaries map produces empty result", func(t *testing.T) { + t.Run("empty extra binaries map produces no extra binary keys", func(t *testing.T) { t.Parallel() config := &UruncConfig{ ExtraBins: map[string]types.ExtraBinConfig{}, @@ -446,7 +447,7 @@ func TestUruncConfigMap(t *testing.T) { cfgMap := config.Map() assert.NotNil(t, cfgMap) - assert.Empty(t, cfgMap) + assert.Equal(t, map[string]string{testLibcontainerKey: "false"}, cfgMap) }) t.Run("vhost true is serialized correctly", func(t *testing.T) { @@ -525,6 +526,7 @@ func TestDefaultConfigs(t *testing.T) { assert.False(t, config.Log.Syslog) assert.False(t, config.Timestamps.Enabled) assert.Equal(t, testTimestampsPath, config.Timestamps.Destination) + assert.False(t, config.Runtime.Libcontainer) assert.Len(t, config.Monitors, 5) assert.Len(t, config.ExtraBins, 1) }) @@ -618,3 +620,68 @@ path = "/usr/bin/mon" assert.Equal(t, defaultMonitorsConfig(), config.Monitors) }) } + +func TestRuntimeLibcontainer(t *testing.T) { + t.Run("absent from the config file defaults to false", func(t *testing.T) { + t.Parallel() + path := writeTestConfig(t, ` +[monitors.qemu] +default_memory_mb = 512 +`) + config, err := LoadUruncConfig(path) + assert.NoError(t, err) + assert.False(t, config.Runtime.Libcontainer) + }) + + t.Run("libcontainer true is honored", func(t *testing.T) { + t.Parallel() + path := writeTestConfig(t, ` +[runtime] +libcontainer = true +`) + + config, err := LoadUruncConfig(path) + assert.NoError(t, err) + assert.True(t, config.Runtime.Libcontainer) + }) + + t.Run("libcontainer false is honored", func(t *testing.T) { + t.Parallel() + path := writeTestConfig(t, ` +[runtime] +libcontainer = false +`) + + config, err := LoadUruncConfig(path) + assert.NoError(t, err) + assert.False(t, config.Runtime.Libcontainer) + }) + + t.Run("survives the round trip through state.json annotations", func(t *testing.T) { + t.Parallel() + path := writeTestConfig(t, ` +[runtime] +libcontainer = true +`) + + config, err := LoadUruncConfig(path) + assert.NoError(t, err) + + cfgMap := config.Map() + assert.Equal(t, "true", cfgMap[testLibcontainerKey]) + + assert.True(t, UruncConfigFromMap(cfgMap).Runtime.Libcontainer) + }) + + t.Run("absent annotation defaults to false", func(t *testing.T) { + t.Parallel() + assert.False(t, UruncConfigFromMap(map[string]string{}).Runtime.Libcontainer) + }) + + t.Run("malformed annotation falls back to false", func(t *testing.T) { + t.Parallel() + cfgMap := map[string]string{testLibcontainerKey: "notBool"} + + assert.False(t, UruncConfigFromMap(cfgMap).Runtime.Libcontainer) + }) +} diff --git a/pkg/unikontainers/utils.go b/pkg/unikontainers/utils.go index 90b9500cf..68977dd0d 100644 --- a/pkg/unikontainers/utils.go +++ b/pkg/unikontainers/utils.go @@ -44,12 +44,16 @@ const ( rootfsDirName = "rootfs" ) -// monitorResources holds the mounts and devices that must be replicated inside -// the monitor's rootfs, along with the block parameters the guest needs. +// monitorResources holds the chosen guest rootfs params, the mounts and devices +// that must be replicated inside the monitor's rootfs, and the block parameters +// the guest needs. type monitorResources struct { - Mounts []specs.Mount `json:"mounts"` - Devices []specs.LinuxDevice `json:"devices"` - BlockArgs []types.BlockDevParams `json:"blockArgs"` + Rootfs types.RootfsParams `json:"rootfs"` + Mounts []specs.Mount `json:"mounts"` + Devices []specs.LinuxDevice `json:"devices"` + BlockArgs []types.BlockDevParams `json:"blockArgs"` + Sharedfs types.SharedfsParams `json:"sharedfs"` + PreStartCmd []string `json:"preStartCmd,omitempty"` } // saveMonitorResources stores the monitorResources passed as an argument in a JSON @@ -243,14 +247,20 @@ func convertUint32ToIntSlice(valSlice []uint32, size int) []int { // return data.Bytes(), nil // } -func spawnProcess(binaryPath string, args []string) error { - cmd := exec.Command(binaryPath, args...) +// spawnProcess starts the process described by argv, whose first element is the +// binary. An empty argv is a no-op. +func spawnProcess(argv []string) error { + if len(argv) == 0 { + return nil + } + // argv is built by urunc, based on its config, not untrusted input + cmd := exec.Command(argv[0], argv[1:]...) //nolint:gosec cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Start(); err != nil { - return err + return fmt.Errorf("failed to start %s: %w", argv[0], err) } return nil