[Qemu-ppc] [PATCH v3 30/35] spapr/xive, xics: reset KVM at machine reset

[Cédric Le Goater]

The interrupt controller KVM device needs to be destroyed and then
recreated accordingly with the interrupt mode negociated at CAS time.
A new KVM_DESTROY_DEVICE is required for this purpose along with the
necessary support in Linux/KVM.

This won't work without the vpcus being first disconnected from the
KVM device.

Signed-off-by: Cédric Le Goater <address@hidden>
---
 hw/intc/spapr_xive_kvm.c    | 39 +++++++++++++++++++++++++++++++++++++++
 hw/intc/xics_kvm.c          | 38 ++++++++++++++++++++++++++++++++++++++
 hw/ppc/spapr.c              | 38 +++++++++++++++++++++++++++-----------
 hw/ppc/spapr_rtas.c         |  2 --
 include/hw/ppc/spapr_xive.h |  1 +
 include/hw/ppc/xics.h       |  1 +
 linux-headers/linux/kvm.h   |  2 ++
 7 files changed, 108 insertions(+), 13 deletions(-)

diff --git a/hw/intc/spapr_xive_kvm.c b/hw/intc/spapr_xive_kvm.c
index e3851991653e..be7c9d1fe0aa 100644
--- a/hw/intc/spapr_xive_kvm.c
+++ b/hw/intc/spapr_xive_kvm.c
@@ -614,3 +614,42 @@ void xive_kvm_init(sPAPRXive *xive, Error **errp)
     kvm_msi_via_irqfd_allowed = true;
     kvm_gsi_direct_mapping = true;
 }
+
+int xive_kvm_fini(sPAPRXive *xive, Error **errp)
+{
+    int rc;
+    XiveSource *xsrc = &xive->source;;
+    struct kvm_create_device xive_destroy_device = {
+        .fd = kernel_xive_fd,
+        .type = KVM_DEV_TYPE_XIVE,
+        .flags = 0,
+    };
+
+    if (kernel_xive_fd == -1) {
+        return 0;
+    }
+
+    if (!kvm_enabled() || !kvmppc_has_cap_xive()) {
+        error_setg(errp,
+                   "KVM and IRQ_XIVE capability must be present for KVM XIVE 
device");
+        return -1;
+    }
+
+    munmap(xsrc->esb_mmap, (1ull << xsrc->esb_shift) * xsrc->nr_irqs);
+    munmap(xive->tm_mmap_user, 1ull << TM_SHIFT);
+    munmap(xive->tm_mmap_os, 1ull << TM_SHIFT);
+
+    rc = kvm_vm_ioctl(kvm_state, KVM_DESTROY_DEVICE, &xive_destroy_device);
+    if (rc < 0) {
+        error_setg_errno(errp, -rc, "Error on KVM_DESTROY_DEVICE for XIVE");
+    }
+    close(xive->fd);
+    xive->fd = -1;
+
+    kernel_xive_fd = -1;
+    kvm_kernel_irqchip = false;
+    kvm_msi_via_irqfd_allowed = false;
+    kvm_gsi_direct_mapping = false;
+
+    return 0;
+}
diff --git a/hw/intc/xics_kvm.c b/hw/intc/xics_kvm.c
index 62ea4ea150f2..0a1b0a703451 100644
--- a/hw/intc/xics_kvm.c
+++ b/hw/intc/xics_kvm.c
@@ -518,6 +518,44 @@ fail:
     return -1;
 }
 
+int xics_kvm_fini(sPAPRMachineState *spapr, Error **errp)
+{
+    int rc;
+    struct kvm_create_device xics_create_device = {
+        .fd = kernel_xics_fd,
+        .type = KVM_DEV_TYPE_XICS,
+        .flags = 0,
+    };
+
+    if (kernel_xics_fd == -1) {
+        return 0;
+    }
+
+    if (!kvm_enabled() || !kvm_check_extension(kvm_state, KVM_CAP_IRQ_XICS)) {
+        error_setg(errp,
+                   "KVM and IRQ_XICS capability must be present for KVM XICS 
device");
+        return -1;
+    }
+
+    rc = kvm_vm_ioctl(kvm_state, KVM_DESTROY_DEVICE, &xics_create_device);
+    if (rc < 0) {
+        error_setg_errno(errp, -rc, "Error on KVM_DESTROY_DEVICE for XICS");
+    }
+    close(kernel_xics_fd);
+    kernel_xics_fd = -1;
+
+    kvmppc_define_rtas_kernel_token(0, "ibm,set-xive");
+    kvmppc_define_rtas_kernel_token(0, "ibm,get-xive");
+    kvmppc_define_rtas_kernel_token(0, "ibm,int-on");
+    kvmppc_define_rtas_kernel_token(0, "ibm,int-off");
+
+    kvm_kernel_irqchip = false;
+    kvm_msi_via_irqfd_allowed = false;
+    kvm_gsi_direct_mapping = false;
+
+    return rc;
+}
+
 static void xics_kvm_register_types(void)
 {
     type_register_static(&ics_kvm_info);
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index c98ceeed9d6f..dea636f9befe 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -1048,9 +1048,8 @@ static void 
spapr_dt_ov5_platform_support(sPAPRMachineState *spapr, void *fdt,
         } else {
             val[3] = 0x00; /* Hash */
         }
-        /* TODO: when under KVM, only advertise XIVE but not both mode */
         if (spapr->xive_exploitation && kvmppc_has_cap_xive()) {
-            val[1] = 0x40; /* OV5_XIVE_EXPLOIT */
+            val[1] = 0x80; /* OV5_XIVE_BOTH */
         }
     } else {
         if (spapr->xive_exploitation) {
@@ -1536,6 +1535,7 @@ static int spapr_reset_drcs(Object *child, void *opaque)
 /* Setup XIVE exploitation or legacy mode as required by CAS */
 static void spapr_reset_interrupt(sPAPRMachineState *spapr, Error **errp)
 {
+    MachineState *machine = MACHINE(spapr);
     Error *local_err = NULL;
     const char *intc_type;
 
@@ -1551,13 +1551,38 @@ static void spapr_reset_interrupt(sPAPRMachineState 
*spapr, Error **errp)
         return;
     }
 
+    /* Destroy KVM device */
+    if (kvm_enabled()) {
+        xics_kvm_fini(spapr, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+        xive_kvm_fini(spapr->xive, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+
     if (spapr_ovec_test(spapr->ov5_cas, OV5_XIVE_EXPLOIT)) {
+        if (kvm_enabled() && machine_kernel_irqchip_allowed(machine)) {
+            xive_kvm_init(spapr->xive, &local_err);
+        }
         spapr_xive_mmio_map(spapr->xive);
         intc_type = spapr->nvt_type;
     } else {
+        if (kvm_enabled() && machine_kernel_irqchip_allowed(machine)) {
+            xics_kvm_init(spapr, &local_err);
+        }
         intc_type = spapr->icp_type;
     }
 
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
     spapr_cpu_core_set_icp(intc_type, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
@@ -2583,15 +2608,6 @@ static void spapr_machine_init(MachineState *machine)
         /* XIVE uses the full range of IRQ numbers. */
         xive_system_init(machine, XICS_IRQ_BASE + XICS_IRQS_SPAPR,
                          &error_fatal);
-
-        /* TODO: initialize KVM for XIVE or for XICS but not for both */
-        if (kvm_enabled() && machine_kernel_irqchip_allowed(machine)) {
-            xive_kvm_init(spapr->xive, &error_fatal);
-        }
-    } else {
-        if (kvm_enabled() && machine_kernel_irqchip_allowed(machine)) {
-            xics_kvm_init(spapr, &error_fatal);
-        }
     }
 
     /* Set up containers for ibm,client-architecture-support negotiated options
diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c
index 0ec5fa4cfe43..9a3d42486e50 100644
--- a/hw/ppc/spapr_rtas.c
+++ b/hw/ppc/spapr_rtas.c
@@ -404,8 +404,6 @@ void spapr_rtas_register(int token, const char *name, 
spapr_rtas_fn fn)
 
     token -= RTAS_TOKEN_BASE;
 
-    assert(!rtas_table[token].name);
-
     rtas_table[token].name = name;
     rtas_table[token].fn = fn;
 }
diff --git a/include/hw/ppc/spapr_xive.h b/include/hw/ppc/spapr_xive.h
index 75b790cc9730..1a28ab0de46d 100644
--- a/include/hw/ppc/spapr_xive.h
+++ b/include/hw/ppc/spapr_xive.h
@@ -88,5 +88,6 @@ void spapr_dt_xive(sPAPRMachineState *spapr, int nr_servers, 
void *fdt,
     OBJECT_CHECK(XiveNVT, (obj), TYPE_XIVE_NVT_KVM)
 
 void xive_kvm_init(sPAPRXive *xive, Error **errp);
+int xive_kvm_fini(sPAPRXive *xive, Error **errp);
 
 #endif /* PPC_SPAPR_XIVE_H */
diff --git a/include/hw/ppc/xics.h b/include/hw/ppc/xics.h
index 6cebff47a7d4..dc2b1bf7ac44 100644
--- a/include/hw/ppc/xics.h
+++ b/include/hw/ppc/xics.h
@@ -204,6 +204,7 @@ void icp_resend(ICPState *ss);
 
 typedef struct sPAPRMachineState sPAPRMachineState;
 
+int xics_kvm_fini(sPAPRMachineState *spapr, Error **errp);
 int xics_kvm_init(sPAPRMachineState *spapr, Error **errp);
 void xics_spapr_init(sPAPRMachineState *spapr);
 
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 2c20d34f194b..8864b855c08b 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1279,6 +1279,8 @@ struct kvm_s390_ucas_mapping {
 #define KVM_GET_DEVICE_ATTR      _IOW(KVMIO,  0xe2, struct kvm_device_attr)
 #define KVM_HAS_DEVICE_ATTR      _IOW(KVMIO,  0xe3, struct kvm_device_attr)
 
+#define KVM_DESTROY_DEVICE       _IOWR(KVMIO,  0xf0, struct kvm_create_device)
+
 /*
  * ioctls for vcpu fds
  */
-- 
2.13.6