From f3407cac7f8a1503ccda5bf45ba18c60b92f929f Mon Sep 17 00:00:00 2001 From: Sharan Raj T Date: Wed, 26 Aug 2026 12:12:24 +0530 Subject: [PATCH] qemu: support `nestedVirtualization` Add support for `nestedVirtualization` in the QEMU driver: - On macOS (aarch64, HVF): requires QEMU 11.1.0 or later and macOS 15 or later (Apple M3 or later). QEMU 11.1 added nested virtualization support to HVF for the aarch64 `virt` machine using `hv_vm_config_set_el2_enabled` (https://wiki.qemu.org/ChangeLog/11.1). Enabled by appending `virtualization=on` to the `-machine virt` option. - On Linux (aarch64, KVM): requires QEMU 10.1.0 or later and a host kernel (6.16 or later) booted with `kvm-arm.mode=nested`. Enabled by appending `virtualization=on` to the `-machine virt` option. - On Linux (x86_64, KVM): verifies that nested virtualization is enabled in the host KVM module (`kvm_intel` or `kvm_amd` parameter `nested`) and that CPU type `host` is used. For unsupported architectures or accelerators, a warning is logged and the field is ignored. Tested on Apple M4 / macOS 26.5 with QEMU 11.1.0: the guest kernel initializes KVM in nVHE mode and a nested KVM VM boots EDK2 to the UEFI shell. Fix #5419 Signed-off-by: Sharan Raj T --- pkg/driver/qemu/qemu.go | 25 ++++++++++ pkg/driver/qemu/qemu_darwin_amd64.go | 18 ++++++++ pkg/driver/qemu/qemu_darwin_arm64.go | 35 ++++++++++++++ pkg/driver/qemu/qemu_driver.go | 12 +++++ pkg/driver/qemu/qemu_linux_amd64.go | 56 +++++++++++++++++++++++ pkg/driver/qemu/qemu_linux_arm64.go | 35 ++++++++++++++ pkg/driver/qemu/qemu_others.go | 18 ++++++++ pkg/driver/qemu/qemu_test.go | 68 ++++++++++++++++++++++++++++ templates/default.yaml | 7 ++- 9 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 pkg/driver/qemu/qemu_darwin_amd64.go create mode 100644 pkg/driver/qemu/qemu_darwin_arm64.go create mode 100644 pkg/driver/qemu/qemu_linux_amd64.go create mode 100644 pkg/driver/qemu/qemu_linux_arm64.go create mode 100644 pkg/driver/qemu/qemu_others.go diff --git a/pkg/driver/qemu/qemu.go b/pkg/driver/qemu/qemu.go index 6693d9f5..3910e0a1 100644 --- a/pkg/driver/qemu/qemu.go +++ b/pkg/driver/qemu/qemu.go @@ -88,6 +88,18 @@ func minimumQemuVersion() (hardMin, softMin semver.Version) { return hardMin, softMin } +// parseKVMNestedParam parses the `nested` module parameter of kvm_intel (bool: "Y"/"N") or kvm_amd (int: "1"/"0"). +func parseKVMNestedParam(s string) (bool, error) { + switch v := strings.TrimSpace(s); v { + case "Y", "y", "1": + return true, nil + case "N", "n", "0": + return false, nil + default: + return false, fmt.Errorf("unexpected value %#q of the `nested` parameter", v) + } +} + // EnsureDisk creates the VM disk from the downloaded image. // For ISO images, it renames the image to "iso" and creates an empty qcow2 disk. // For non-ISO images, it validates and renames the image to "disk". @@ -550,6 +562,16 @@ func Cmdline(ctx context.Context, cfg Config) (exe string, args []string, err er } args = appendArgsIfNoConflict(args, "-cpu", cpu) + // Nested virtualization + nestedVirtEnabled := false + if y.NestedVirtualization != nil && *y.NestedVirtualization { + var err error + nestedVirtEnabled, err = nestedVirtualizationEnabled(*y.Arch, accel, cpu, version) + if err != nil { + return "", nil, err + } + } + // Machine switch *y.Arch { case limatype.X8664: @@ -570,6 +592,9 @@ func Cmdline(ctx context.Context, cfg Config) (exe string, args []string, err er } case limatype.AARCH64: machine := "virt,accel=" + accel + if nestedVirtEnabled { + machine += ",virtualization=on" + } args = appendArgsIfNoConflict(args, "-machine", machine) case limatype.RISCV64: // https://github.com/tianocore/edk2/blob/edk2-stable202408/OvmfPkg/RiscVVirt/README.md#test diff --git a/pkg/driver/qemu/qemu_darwin_amd64.go b/pkg/driver/qemu/qemu_darwin_amd64.go new file mode 100644 index 00000000..863839ca --- /dev/null +++ b/pkg/driver/qemu/qemu_darwin_amd64.go @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright The Lima Authors +// SPDX-License-Identifier: Apache-2.0 + +package qemu + +import ( + "github.com/coreos/go-semver/semver" + "github.com/sirupsen/logrus" +) + +// nestedVirtualizationEnabled reports whether the host can provide nested virtualization +// for the given guest configuration. +// +// On Intel Macs, HVF does not support nested virtualization. +func nestedVirtualizationEnabled(_, _, _ string, _ *semver.Version) (bool, error) { + logrus.Warn("field `nestedVirtualization` is not supported on this platform, ignoring") + return false, nil +} diff --git a/pkg/driver/qemu/qemu_darwin_arm64.go b/pkg/driver/qemu/qemu_darwin_arm64.go new file mode 100644 index 00000000..3a7bb3d8 --- /dev/null +++ b/pkg/driver/qemu/qemu_darwin_arm64.go @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright The Lima Authors +// SPDX-License-Identifier: Apache-2.0 + +package qemu + +import ( + "fmt" + "strings" + + "github.com/coreos/go-semver/semver" + "github.com/sirupsen/logrus" + + "github.com/lima-vm/lima/v2/pkg/limatype" +) + +// nestedVirtualizationEnabled reports whether the host can provide nested virtualization +// for the given guest configuration. +// +// On aarch64 macOS, HVF nested virtualization requires QEMU 11.1.0 or later. +// https://wiki.qemu.org/ChangeLog/11.1 +func nestedVirtualizationEnabled(arch, accel, _ string, qemuVer *semver.Version) (bool, error) { + if arch != limatype.AARCH64 { + logrus.Warnf("field `nestedVirtualization` is not supported for architecture %#q, ignoring", arch) + return false, nil + } + if accel != "hvf" { + logrus.Warnf("nested virtualization is not supported with accelerator %s on this platform, ignoring", strings.ToUpper(accel)) + return false, nil + } + minVer := semver.New("11.1.0") + if qemuVer != nil && qemuVer.LessThan(*minVer) { + return false, fmt.Errorf("QEMU %v is too old, %v or later is required for nested virtualization with HVF", qemuVer, minVer) + } + return true, nil +} diff --git a/pkg/driver/qemu/qemu_driver.go b/pkg/driver/qemu/qemu_driver.go index db98bd86..d5e7b5c7 100644 --- a/pkg/driver/qemu/qemu_driver.go +++ b/pkg/driver/qemu/qemu_driver.go @@ -139,6 +139,18 @@ func validateConfig(cfg *limatype.LimaYAML) error { } } + if cfg.NestedVirtualization != nil && *cfg.NestedVirtualization { + if runtime.GOOS == "darwin" { + macOSProductVersion, err := osutil.ProductVersion() + if err != nil { + return err + } + if macOSProductVersion.LessThan(*semver.New("15.0.0")) { + return errors.New("nested virtualization requires macOS 15 or newer") + } + } + } + return nil } diff --git a/pkg/driver/qemu/qemu_linux_amd64.go b/pkg/driver/qemu/qemu_linux_amd64.go new file mode 100644 index 00000000..daed5abb --- /dev/null +++ b/pkg/driver/qemu/qemu_linux_amd64.go @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright The Lima Authors +// SPDX-License-Identifier: Apache-2.0 + +package qemu + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/coreos/go-semver/semver" + "github.com/sirupsen/logrus" + + "github.com/lima-vm/lima/v2/pkg/limatype" +) + +// nestedVirtualizationEnabled reports whether the host can provide nested virtualization +// for the given guest configuration. +// +// On x86_64 Linux, nested virtualization is controlled by the host KVM module +// (the `nested` parameter of kvm_intel / kvm_amd) and is exposed to the guest via `-cpu host`. +// No minimum QEMU version is required. +func nestedVirtualizationEnabled(arch, accel, cpu string, _ *semver.Version) (bool, error) { + if arch != limatype.X8664 { + logrus.Warnf("field `nestedVirtualization` is not supported for architecture %#q, ignoring", arch) + return false, nil + } + if accel != "kvm" { + logrus.Warnf("field `nestedVirtualization` is not supported with accelerator %#q for architecture %#q, ignoring", accel, arch) + return false, nil + } + for _, module := range []string{"kvm_intel", "kvm_amd"} { + b, err := os.ReadFile(filepath.Join("/sys/module", module, "parameters", "nested")) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + logrus.WithError(err).Warn("Failed to check whether nested virtualization is enabled in the host KVM module") + return false, nil + } + enabled, err := parseKVMNestedParam(string(b)) + if err != nil { + return false, err + } + if !enabled { + return false, errors.New("nested virtualization is disabled in the host KVM module (`nested` parameter of kvm_intel / kvm_amd)") + } + if !strings.HasPrefix(cpu, "host") { + logrus.Warnf("nested virtualization on x86_64 requires CPU type `host`, got %#q", cpu) + } + return true, nil + } + return false, errors.New("neither kvm_intel nor kvm_amd is loaded") +} diff --git a/pkg/driver/qemu/qemu_linux_arm64.go b/pkg/driver/qemu/qemu_linux_arm64.go new file mode 100644 index 00000000..9f51b75a --- /dev/null +++ b/pkg/driver/qemu/qemu_linux_arm64.go @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright The Lima Authors +// SPDX-License-Identifier: Apache-2.0 + +package qemu + +import ( + "fmt" + "strings" + + "github.com/coreos/go-semver/semver" + "github.com/sirupsen/logrus" + + "github.com/lima-vm/lima/v2/pkg/limatype" +) + +// nestedVirtualizationEnabled reports whether the host can provide nested virtualization +// for the given guest configuration. +// +// On aarch64 Linux, KVM_CAP_ARM_EL2 requires QEMU 10.1.0 or later and a host kernel +// (6.16 or later) booted with `kvm-arm.mode=nested`. +func nestedVirtualizationEnabled(arch, accel, _ string, qemuVer *semver.Version) (bool, error) { + if arch != limatype.AARCH64 { + logrus.Warnf("field `nestedVirtualization` is not supported for architecture %#q, ignoring", arch) + return false, nil + } + if accel != "kvm" { + logrus.Warnf("nested virtualization is not supported with accelerator %s on this platform, ignoring", strings.ToUpper(accel)) + return false, nil + } + minVer := semver.New("10.1.0") + if qemuVer != nil && qemuVer.LessThan(*minVer) { + return false, fmt.Errorf("QEMU %v is too old, %v or later is required for nested virtualization with KVM", qemuVer, minVer) + } + return true, nil +} diff --git a/pkg/driver/qemu/qemu_others.go b/pkg/driver/qemu/qemu_others.go new file mode 100644 index 00000000..d5f0d165 --- /dev/null +++ b/pkg/driver/qemu/qemu_others.go @@ -0,0 +1,18 @@ +//go:build !(linux || darwin) || !(amd64 || arm64) + +// SPDX-FileCopyrightText: Copyright The Lima Authors +// SPDX-License-Identifier: Apache-2.0 + +package qemu + +import ( + "github.com/coreos/go-semver/semver" + "github.com/sirupsen/logrus" +) + +// nestedVirtualizationEnabled reports whether the host can provide nested virtualization +// for the given guest configuration. +func nestedVirtualizationEnabled(_, _, _ string, _ *semver.Version) (bool, error) { + logrus.Warn("field `nestedVirtualization` is not supported on this platform, ignoring") + return false, nil +} diff --git a/pkg/driver/qemu/qemu_test.go b/pkg/driver/qemu/qemu_test.go index 919effe6..6b21a7a0 100644 --- a/pkg/driver/qemu/qemu_test.go +++ b/pkg/driver/qemu/qemu_test.go @@ -9,10 +9,12 @@ import ( "runtime" "testing" + "github.com/coreos/go-semver/semver" "gotest.tools/v3/assert" "github.com/lima-vm/lima/v2/pkg/limatype" "github.com/lima-vm/lima/v2/pkg/limatype/filenames" + "github.com/lima-vm/lima/v2/pkg/osutil" ) func TestArgValue(t *testing.T) { @@ -151,3 +153,69 @@ func TestSwtpmCmdline(t *testing.T) { _, err = os.Stat(swtpmSock) assert.ErrorIs(t, err, os.ErrNotExist) } + +func TestParseKVMNestedParam(t *testing.T) { + type testCase struct { + value string + expected bool + expectedError string + } + testCases := []testCase{ + {value: "Y\n", expected: true}, // kvm_intel (bool) + {value: "N\n", expected: false}, // kvm_intel (bool) + {value: "1\n", expected: true}, // kvm_amd (int) + {value: "0\n", expected: false}, // kvm_amd (int) + {value: "", expectedError: "unexpected value"}, + {value: "foo", expectedError: "unexpected value"}, + } + for _, tc := range testCases { + enabled, err := parseKVMNestedParam(tc.value) + if tc.expectedError == "" { + assert.NilError(t, err) + } else { + assert.ErrorContains(t, err, tc.expectedError) + } + assert.Equal(t, tc.expected, enabled) + } +} + +func TestValidateConfigNestedVirtualization(t *testing.T) { + cfg := &limatype.LimaYAML{ + NestedVirtualization: new(true), + } + err := validateConfig(cfg) + if runtime.GOOS == "darwin" { + productVer, verErr := osutil.ProductVersion() + assert.NilError(t, verErr) + if productVer.LessThan(*semver.New("15.0.0")) { + assert.ErrorContains(t, err, "nested virtualization requires macOS 15 or newer") + } else { + assert.NilError(t, err) + } + } else { + assert.NilError(t, err) + } +} + +func TestNestedVirtualizationEnabled(t *testing.T) { + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + // Unsupported arch + enabled, err := nestedVirtualizationEnabled(limatype.X8664, "hvf", "host", semver.New("11.1.0")) + assert.NilError(t, err) + assert.Equal(t, false, enabled) + + // Unsupported accel + enabled, err = nestedVirtualizationEnabled(limatype.AARCH64, "tcg", "cortex-a72", semver.New("11.1.0")) + assert.NilError(t, err) + assert.Equal(t, false, enabled) + + // QEMU too old + _, err = nestedVirtualizationEnabled(limatype.AARCH64, "hvf", "host", semver.New("11.0.0")) + assert.ErrorContains(t, err, "QEMU 11.0.0 is too old") + + // Supported + enabled, err = nestedVirtualizationEnabled(limatype.AARCH64, "hvf", "host", semver.New("11.1.0")) + assert.NilError(t, err) + assert.Equal(t, true, enabled) + } +} diff --git a/templates/default.yaml b/templates/default.yaml index 430f66ad..bd00a66f 100644 --- a/templates/default.yaml +++ b/templates/default.yaml @@ -665,7 +665,12 @@ plain: null # qemu-system-aarch64 -accel kvm -cpu host -M virt # - Without specifying `-cpu host`, nested virtualization may fail with the error: # qemu-system-aarch64: kvm_init_vcpu: kvm_arch_init_vcpu failed (0): Invalid argument -# - Only supported on Apple M3 or later with `vmType: vz` or `vmType: krunkit`. +# - Supported with `vmType: vz` or `vmType: krunkit` on Apple M3 or later (macOS 15 or later). +# - Supported with `vmType: qemu` for aarch64 guests (`-machine virt,virtualization=on`): +# - macOS (HVF): QEMU 11.1 or later, on Apple M3 or later (macOS 15 or later). +# - Linux (KVM): QEMU 10.1 or later, and a host kernel (6.16 or later) booted with `kvm-arm.mode=nested`. +# - For x86_64 guests with `vmType: qemu` on Linux (KVM), nested virtualization is controlled by the host +# (the `nested` parameter of the kvm_intel / kvm_amd module) and is exposed to the guest via `-cpu host`. # 🟢 Builtin default: false nestedVirtualization: null