qemu: support nestedVirtualization

Add support for `nestedVirtualization` in the QEMU driver:

- On macOS (aarch64, HVF): requires QEMU 11.1.0 or later and macOS 15 or
  later (Apple M3 or later). QEMU 11.1 added nested virtualization support
  to HVF for the aarch64 `virt` machine using `hv_vm_config_set_el2_enabled`
  (https://wiki.qemu.org/ChangeLog/11.1). Enabled by appending
  `virtualization=on` to the `-machine virt` option.
- On Linux (aarch64, KVM): requires QEMU 10.1.0 or later and a host kernel
  (6.16 or later) booted with `kvm-arm.mode=nested`. Enabled by appending
  `virtualization=on` to the `-machine virt` option.
- On Linux (x86_64, KVM): verifies that nested virtualization is enabled
  in the host KVM module (`kvm_intel` or `kvm_amd` parameter `nested`) and
  that CPU type `host` is used.

For unsupported architectures or accelerators, a warning is logged and
the field is ignored.

Tested on Apple M4 / macOS 26.5 with QEMU 11.1.0: the guest kernel
initializes KVM in nVHE mode and a nested KVM VM boots EDK2 to the
UEFI shell.

Fix #5419

Signed-off-by: Sharan Raj T <sharanrajtm@gmail.com>

pkg/driver/qemu/qemu.go

@@ -88,6 +88,18 @@ func minimumQemuVersion() (hardMin, softMin semver.Version) {
 	return hardMin, softMin
 }
 
+// parseKVMNestedParam parses the `nested` module parameter of kvm_intel (bool: "Y"/"N") or kvm_amd (int: "1"/"0").
+func parseKVMNestedParam(s string) (bool, error) {
+	switch v := strings.TrimSpace(s); v {
+	case "Y", "y", "1":
+		return true, nil
+	case "N", "n", "0":
+		return false, nil
+	default:
+		return false, fmt.Errorf("unexpected value %#q of the `nested` parameter", v)
+	}
+}
+
 // EnsureDisk creates the VM disk from the downloaded image.
 // For ISO images, it renames the image to "iso" and creates an empty qcow2 disk.
 // For non-ISO images, it validates and renames the image to "disk".
@@ -550,6 +562,16 @@ func Cmdline(ctx context.Context, cfg Config) (exe string, args []string, err er
 	}
 	args = appendArgsIfNoConflict(args, "-cpu", cpu)
 
+	// Nested virtualization
+	nestedVirtEnabled := false
+	if y.NestedVirtualization != nil && *y.NestedVirtualization {
+		var err error
+		nestedVirtEnabled, err = nestedVirtualizationEnabled(*y.Arch, accel, cpu, version)
+		if err != nil {
+			return "", nil, err
+		}
+	}
+
 	// Machine
 	switch *y.Arch {
 	case limatype.X8664:
@@ -570,6 +592,9 @@ func Cmdline(ctx context.Context, cfg Config) (exe string, args []string, err er
 		}
 	case limatype.AARCH64:
 		machine := "virt,accel=" + accel
+		if nestedVirtEnabled {
+			machine += ",virtualization=on"
+		}
 		args = appendArgsIfNoConflict(args, "-machine", machine)
 	case limatype.RISCV64:
 		// https://github.com/tianocore/edk2/blob/edk2-stable202408/OvmfPkg/RiscVVirt/README.md#test

pkg/driver/qemu/qemu_darwin_amd64.go

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package qemu
+
+import (
+	"github.com/coreos/go-semver/semver"
+	"github.com/sirupsen/logrus"
+)
+
+// nestedVirtualizationEnabled reports whether the host can provide nested virtualization
+// for the given guest configuration.
+//
+// On Intel Macs, HVF does not support nested virtualization.
+func nestedVirtualizationEnabled(_, _, _ string, _ *semver.Version) (bool, error) {
+	logrus.Warn("field `nestedVirtualization` is not supported on this platform, ignoring")
+	return false, nil
+}

pkg/driver/qemu/qemu_darwin_arm64.go

@@ -0,0 +1,35 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package qemu
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/coreos/go-semver/semver"
+	"github.com/sirupsen/logrus"
+
+	"github.com/lima-vm/lima/v2/pkg/limatype"
+)
+
+// nestedVirtualizationEnabled reports whether the host can provide nested virtualization
+// for the given guest configuration.
+//
+// On aarch64 macOS, HVF nested virtualization requires QEMU 11.1.0 or later.
+// https://wiki.qemu.org/ChangeLog/11.1
+func nestedVirtualizationEnabled(arch, accel, _ string, qemuVer *semver.Version) (bool, error) {
+	if arch != limatype.AARCH64 {
+		logrus.Warnf("field `nestedVirtualization` is not supported for architecture %#q, ignoring", arch)
+		return false, nil
+	}
+	if accel != "hvf" {
+		logrus.Warnf("nested virtualization is not supported with accelerator %s on this platform, ignoring", strings.ToUpper(accel))
+		return false, nil
+	}
+	minVer := semver.New("11.1.0")
+	if qemuVer != nil && qemuVer.LessThan(*minVer) {
+		return false, fmt.Errorf("QEMU %v is too old, %v or later is required for nested virtualization with HVF", qemuVer, minVer)
+	}
+	return true, nil
+}

pkg/driver/qemu/qemu_driver.go

@@ -139,6 +139,18 @@ func validateConfig(cfg *limatype.LimaYAML) error {
 		}
 	}
 
+	if cfg.NestedVirtualization != nil && *cfg.NestedVirtualization {
+		if runtime.GOOS == "darwin" {
+			macOSProductVersion, err := osutil.ProductVersion()
+			if err != nil {
+				return err
+			}
+			if macOSProductVersion.LessThan(*semver.New("15.0.0")) {
+				return errors.New("nested virtualization requires macOS 15 or newer")
+			}
+		}
+	}
+
 	return nil
 }
 

pkg/driver/qemu/qemu_linux_amd64.go

@@ -0,0 +1,56 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package qemu
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/coreos/go-semver/semver"
+	"github.com/sirupsen/logrus"
+
+	"github.com/lima-vm/lima/v2/pkg/limatype"
+)
+
+// nestedVirtualizationEnabled reports whether the host can provide nested virtualization
+// for the given guest configuration.
+//
+// On x86_64 Linux, nested virtualization is controlled by the host KVM module
+// (the `nested` parameter of kvm_intel / kvm_amd) and is exposed to the guest via `-cpu host`.
+// No minimum QEMU version is required.
+func nestedVirtualizationEnabled(arch, accel, cpu string, _ *semver.Version) (bool, error) {
+	if arch != limatype.X8664 {
+		logrus.Warnf("field `nestedVirtualization` is not supported for architecture %#q, ignoring", arch)
+		return false, nil
+	}
+	if accel != "kvm" {
+		logrus.Warnf("field `nestedVirtualization` is not supported with accelerator %#q for architecture %#q, ignoring", accel, arch)
+		return false, nil
+	}
+	for _, module := range []string{"kvm_intel", "kvm_amd"} {
+		b, err := os.ReadFile(filepath.Join("/sys/module", module, "parameters", "nested"))
+		if err != nil {
+			if errors.Is(err, fs.ErrNotExist) {
+				continue
+			}
+			logrus.WithError(err).Warn("Failed to check whether nested virtualization is enabled in the host KVM module")
+			return false, nil
+		}
+		enabled, err := parseKVMNestedParam(string(b))
+		if err != nil {
+			return false, err
+		}
+		if !enabled {
+			return false, errors.New("nested virtualization is disabled in the host KVM module (`nested` parameter of kvm_intel / kvm_amd)")
+		}
+		if !strings.HasPrefix(cpu, "host") {
+			logrus.Warnf("nested virtualization on x86_64 requires CPU type `host`, got %#q", cpu)
+		}
+		return true, nil
+	}
+	return false, errors.New("neither kvm_intel nor kvm_amd is loaded")
+}

pkg/driver/qemu/qemu_linux_arm64.go

@@ -0,0 +1,35 @@
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package qemu
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/coreos/go-semver/semver"
+	"github.com/sirupsen/logrus"
+
+	"github.com/lima-vm/lima/v2/pkg/limatype"
+)
+
+// nestedVirtualizationEnabled reports whether the host can provide nested virtualization
+// for the given guest configuration.
+//
+// On aarch64 Linux, KVM_CAP_ARM_EL2 requires QEMU 10.1.0 or later and a host kernel
+// (6.16 or later) booted with `kvm-arm.mode=nested`.
+func nestedVirtualizationEnabled(arch, accel, _ string, qemuVer *semver.Version) (bool, error) {
+	if arch != limatype.AARCH64 {
+		logrus.Warnf("field `nestedVirtualization` is not supported for architecture %#q, ignoring", arch)
+		return false, nil
+	}
+	if accel != "kvm" {
+		logrus.Warnf("nested virtualization is not supported with accelerator %s on this platform, ignoring", strings.ToUpper(accel))
+		return false, nil
+	}
+	minVer := semver.New("10.1.0")
+	if qemuVer != nil && qemuVer.LessThan(*minVer) {
+		return false, fmt.Errorf("QEMU %v is too old, %v or later is required for nested virtualization with KVM", qemuVer, minVer)
+	}
+	return true, nil
+}

pkg/driver/qemu/qemu_others.go

@@ -0,0 +1,18 @@
+//go:build !(linux || darwin) || !(amd64 || arm64)
+
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package qemu
+
+import (
+	"github.com/coreos/go-semver/semver"
+	"github.com/sirupsen/logrus"
+)
+
+// nestedVirtualizationEnabled reports whether the host can provide nested virtualization
+// for the given guest configuration.
+func nestedVirtualizationEnabled(_, _, _ string, _ *semver.Version) (bool, error) {
+	logrus.Warn("field `nestedVirtualization` is not supported on this platform, ignoring")
+	return false, nil
+}

pkg/driver/qemu/qemu_test.go

@@ -9,10 +9,12 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/coreos/go-semver/semver"
 	"gotest.tools/v3/assert"
 
 	"github.com/lima-vm/lima/v2/pkg/limatype"
 	"github.com/lima-vm/lima/v2/pkg/limatype/filenames"
+	"github.com/lima-vm/lima/v2/pkg/osutil"
 )
 
 func TestArgValue(t *testing.T) {
@@ -151,3 +153,69 @@ func TestSwtpmCmdline(t *testing.T) {
 	_, err = os.Stat(swtpmSock)
 	assert.ErrorIs(t, err, os.ErrNotExist)
 }
+
+func TestParseKVMNestedParam(t *testing.T) {
+	type testCase struct {
+		value         string
+		expected      bool
+		expectedError string
+	}
+	testCases := []testCase{
+		{value: "Y\n", expected: true},  // kvm_intel (bool)
+		{value: "N\n", expected: false}, // kvm_intel (bool)
+		{value: "1\n", expected: true},  // kvm_amd (int)
+		{value: "0\n", expected: false}, // kvm_amd (int)
+		{value: "", expectedError: "unexpected value"},
+		{value: "foo", expectedError: "unexpected value"},
+	}
+	for _, tc := range testCases {
+		enabled, err := parseKVMNestedParam(tc.value)
+		if tc.expectedError == "" {
+			assert.NilError(t, err)
+		} else {
+			assert.ErrorContains(t, err, tc.expectedError)
+		}
+		assert.Equal(t, tc.expected, enabled)
+	}
+}
+
+func TestValidateConfigNestedVirtualization(t *testing.T) {
+	cfg := &limatype.LimaYAML{
+		NestedVirtualization: new(true),
+	}
+	err := validateConfig(cfg)
+	if runtime.GOOS == "darwin" {
+		productVer, verErr := osutil.ProductVersion()
+		assert.NilError(t, verErr)
+		if productVer.LessThan(*semver.New("15.0.0")) {
+			assert.ErrorContains(t, err, "nested virtualization requires macOS 15 or newer")
+		} else {
+			assert.NilError(t, err)
+		}
+	} else {
+		assert.NilError(t, err)
+	}
+}
+
+func TestNestedVirtualizationEnabled(t *testing.T) {
+	if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
+		// Unsupported arch
+		enabled, err := nestedVirtualizationEnabled(limatype.X8664, "hvf", "host", semver.New("11.1.0"))
+		assert.NilError(t, err)
+		assert.Equal(t, false, enabled)
+
+		// Unsupported accel
+		enabled, err = nestedVirtualizationEnabled(limatype.AARCH64, "tcg", "cortex-a72", semver.New("11.1.0"))
+		assert.NilError(t, err)
+		assert.Equal(t, false, enabled)
+
+		// QEMU too old
+		_, err = nestedVirtualizationEnabled(limatype.AARCH64, "hvf", "host", semver.New("11.0.0"))
+		assert.ErrorContains(t, err, "QEMU 11.0.0 is too old")
+
+		// Supported
+		enabled, err = nestedVirtualizationEnabled(limatype.AARCH64, "hvf", "host", semver.New("11.1.0"))
+		assert.NilError(t, err)
+		assert.Equal(t, true, enabled)
+	}
+}

templates/default.yaml

@@ -665,7 +665,12 @@ plain: null
 #   qemu-system-aarch64 -accel kvm -cpu host -M virt
 # - Without specifying `-cpu host`, nested virtualization may fail with the error:
 #   qemu-system-aarch64: kvm_init_vcpu: kvm_arch_init_vcpu failed (0): Invalid argument
-# - Only supported on Apple M3 or later with `vmType: vz` or `vmType: krunkit`.
+# - Supported with `vmType: vz` or `vmType: krunkit` on Apple M3 or later (macOS 15 or later).
+# - Supported with `vmType: qemu` for aarch64 guests (`-machine virt,virtualization=on`):
+#   - macOS (HVF): QEMU 11.1 or later, on Apple M3 or later (macOS 15 or later).
+#   - Linux (KVM): QEMU 10.1 or later, and a host kernel (6.16 or later) booted with `kvm-arm.mode=nested`.
+# - For x86_64 guests with `vmType: qemu` on Linux (KVM), nested virtualization is controlled by the host
+#   (the `nested` parameter of the kvm_intel / kvm_amd module) and is exposed to the guest via `-cpu host`.
 # 🟢 Builtin default: false
 nestedVirtualization: null