Nested virtualization with windows guest

[Michał Zegan]

Hello,


I am running on the host with intel alderlake (12'th gen) cpu which is 
i7 12700h, running fedora 40, linux kernel 6.12.8, qemu 8.2.8.

I run newest windows 11 as a guest, but this problem traces back at 
least to windows 10.

It works, until I enable hyperv. After that, it starts boot looping.

The only way to stop it boot looping is to disable hyperv, or the 
virtualization feature in cpu.

What could be the problem/how to fix it?

I have asked that before, but years are coming, this is not fixed, and 
workarounds I've found on forums don't work.


This is the libvirt log fragment containing qemu cmdline: Note it has 
hardware virtualization disabled, enabling it makes windows boot loop.


2024-08-27-14:14:13, ), qemu version: 8.2.8qemu-8.2.8-2.fc40, kernel: 
6.12.8-100.local.fc40.x86_64, hostname: wlap
LC_ALL=C \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \
USER=root \
HOME=/var/lib/libvirt/qemu/domain-1-win11 \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-1-win11/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-1-win11/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-1-win11/.config \
XDG_RUNTIME_DIR=/run/user/1000 \
/usr/bin/qemu-system-x86_64 \
-name guest=win11,debug-threads=on \
-S \
-object 
'{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-1-win11/master-key.aes"}' 
\
-blockdev 
'{"driver":"file","filename":"/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' 
\
-blockdev 
'{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' 
\
-blockdev 
'{"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/win11_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' 
\
-blockdev 
'{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' 
\
-machine 
pc-q35-7.2,usb=off,smm=on,kernel_irqchip=on,dump-guest-core=on,memory-backend=pc.ram,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,acpi=on 
\
-accel kvm \
-cpu 
host,migratable=off,vmx=off,kvmclock=on,hv-time=on,kvm-pv-eoi=on,kvm-pv-unhalt=off,hv-passthrough=on,hv-crash,kvm-pv-ipi=off,pmu=on 
\
-global driver=cfi.pflash01,property=secure,value=on \
-m size=8388608k \
-object 
'{"qom-type":"memory-backend-memfd","id":"pc.ram","share":true,"x-use-canonical-path-for-ramblock-id":false,"size":8589934592}' 
\
-overcommit mem-lock=off \
-smp 8,sockets=1,dies=1,clusters=1,cores=8,threads=1 \
-uuid 8c101ee1-29e1-43f8-9136-e9b6066ed4d6 \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=23,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime,driftfix=slew \
-no-shutdown \
-boot strict=on \
-device 
'{"driver":"intel-iommu","id":"iommu0","caching-mode":true,"device-iotlb":true}' 
\
-device 
'{"driver":"pcie-root-port","port":16,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x1"}' 
\
-device 
'{"driver":"pcie-root-port","port":17,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x1.0x1"}' 
\
-device 
'{"driver":"pcie-root-port","port":18,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x1.0x2"}' 
\
-device 
'{"driver":"pcie-root-port","port":19,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x1.0x3"}' 
\
-device 
'{"driver":"pcie-root-port","port":20,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x1.0x4"}' 
\
-device 
'{"driver":"pcie-root-port","port":21,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x1.0x5"}' 
\
-device 
'{"driver":"pcie-root-port","port":22,"chassis":7,"id":"pci.7","bus":"pcie.0","addr":"0x1.0x6"}' 
\
-device 
'{"driver":"qemu-xhci","id":"usb","bus":"pci.3","multifunction":true,"addr":"0x0"}' 
\
-device 
'{"driver":"virtio-scsi-pci","iommu_platform":true,"ats":true,"id":"scsi0","num_queues":8,"bus":"pci.3","addr":"0x0.0x1"}' 
\
-device 
'{"driver":"virtio-serial-pci","iommu_platform":true,"ats":true,"id":"virtio-serial0","bus":"pci.3","addr":"0x0.0x2"}' 
\
-device '{"driver":"ide-cd","bus":"ide.1","id":"sata0-0-1","bootindex":2}' \
-blockdev 
'{"driver":"host_device","filename":"/dev/pool/win11","aio":"io_uring","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' 
\
-blockdev 
'{"node-name":"libvirt-1-format","read-only":false,"discard":"unmap","driver":"raw","file":"libvirt-1-storage"}' 
\
-device 
'{"driver":"scsi-hd","bus":"scsi0.0","channel":0,"scsi-id":0,"lun":0,"device_id":"drive-scsi0-0-0-0","drive":"libvirt-1-format","id":"scsi0-0-0-0","bootindex":1}' 
\
-chardev 
socket,id=chr-vu-fs0,path=/var/lib/libvirt/qemu/domain-1-win11/fs0-fs.sock \
-device 
'{"driver":"vhost-user-fs-pci","id":"fs0","chardev":"chr-vu-fs0","tag":"shared","bus":"pci.5","addr":"0x0"}' 
\
-netdev 
'{"type":"tap","fds":"24:26:27:28:29:30:31:32","vhost":true,"vhostfds":"33:34:35:36:37:38:39:40","id":"hostnet0"}' 
\
-device 
'{"driver":"virtio-net-pci","iommu_platform":true,"ats":true,"mq":true,"vectors":18,"netdev":"hostnet0","id":"net0","mac":"52:54:00:4e:aa:fc","bootindex":3,"bus":"pci.4","addr":"0x0","rombar":1,"romfile":"/usr/share/ipxe/qemu/efi-virtio.rom"}' 
\
-chardev pty,id=charserial0 \
-device 
'{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \
-chardev spicevmc,id=charchannel0,name=vdagent \
-device 
'{"driver":"virtserialport","bus":"virtio-serial0.0","nr":1,"chardev":"charchannel0","id":"channel0","name":"com.redhat.spice.0"}' 
\
-chardev socket,id=charchannel1,fd=22,server=on,wait=off \
-device 
'{"driver":"virtserialport","bus":"virtio-serial0.0","nr":2,"chardev":"charchannel1","id":"channel1","name":"org.qemu.guest_agent.0"}' 
\
-chardev socket,id=chrtpm,path=/run/libvirt/qemu/swtpm/1-win11-swtpm.sock \
-tpmdev emulator,id=tpm-tpm0,chardev=chrtpm \
-device '{"driver":"tpm-crb","tpmdev":"tpm-tpm0","id":"tpm0"}' \
-object 
'{"qom-type":"input-linux","id":"input0","evdev":"/dev/input/by-id/usb-MOSART_Semi._2.4G_INPUT_DEVICE-event-kbd","repeat":true,"grab_all":true,"grab-toggle":"ctrl-ctrl"}' 
\
-object 
'{"qom-type":"input-linux","id":"input1","evdev":"/dev/input/by-path/platform-i8042-serio-0-event-kbd","repeat":true,"grab_all":true,"grab-toggle":"ctrl-ctrl"}' 
\
-object 
'{"qom-type":"input-linux","id":"input2","evdev":"/dev/input/by-path/pci-0000:00:15.0-platform-i2c_designware.0-event-mouse"}' 
\
-device '{"driver":"usb-tablet","id":"input5","bus":"usb.0","port":"3"}' \
-device '{"driver":"usb-kbd","id":"input6","bus":"usb.0","port":"4"}' \
-audiodev '{"id":"audio1","driver":"pa"}' \
-spice 
port=0,disable-ticketing=on,gl=on,rendernode=/dev/dri/renderD128,seamless-migration=on 
\
-device 
'{"driver":"virtio-vga","id":"video0","max_outputs":1,"bus":"pci.2","multifunction":true,"addr":"0x0"}' 
\
-device 
'{"driver":"ich9-intel-hda","id":"sound0","bus":"pci.2","addr":"0x0.0x1"}' \
-device 
'{"driver":"hda-micro","id":"sound0-codec0","bus":"sound0.0","cad":0,"audiodev":"audio1"}' 
\
-device 
'{"driver":"i6300esb","id":"watchdog0","bus":"pci.1","multifunction":true,"addr":"0x0"}' 
\
-global ICH9-LPC.noreboot=off \
-watchdog-action reset \
-chardev spicevmc,id=charredir0,name=usbredir \
-device 
'{"driver":"usb-redir","chardev":"charredir0","id":"redir0","bus":"usb.0","port":"1"}' 
\
-chardev spicevmc,id=charredir1,name=usbredir \
-device 
'{"driver":"usb-redir","chardev":"charredir1","id":"redir1","bus":"usb.0","port":"2"}' 
\
-device 
'{"driver":"virtio-balloon-pci","iommu_platform":true,"ats":true,"id":"balloon0","deflate-on-oom":true,"free-page-reporting":true,"bus":"pci.1","addr":"0x0.0x1"}' 
\
-object '{"qom-type":"rng-builtin","id":"objrng0"}' \
-device 
'{"driver":"virtio-rng-pci","iommu_platform":true,"ats":true,"rng":"objrng0","id":"rng0","bus":"pci.1","addr":"0x0.0x2"}' 
\
-device '{"driver":"vmcoreinfo"}' \
-sandbox 
on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-device '{"driver":"pvpanic"}' \
-msg timestamp=on