[RFC PATCH v4 2/2] s390x: Implement the USER_BUSY capability

[Eric Farman]

With the USER_SIGP capability, the kernel will pass most (but not all)
SIGP orders to userspace for processing. But that means that the kernel
is unable to determine if/when the order has been completed by userspace,
and could potentially return an incorrect answer (CC1 with status bits
versus CC2 indicating BUSY) for one of the remaining in-kernel orders.

With a new USER_BUSY capability, userspace can tell the kernel when
it is started processing a SIGP order and when it has finished, such that
the in-kernel orders can be returned with the BUSY condition between the
two IOCTLs.

Signed-off-by: Eric Farman <farman@linux.ibm.com>
---
 target/s390x/cpu-sysemu.c    | 15 ++++++++++++++
 target/s390x/cpu.c           |  1 +
 target/s390x/cpu.h           |  8 ++++++++
 target/s390x/kvm/kvm.c       | 38 ++++++++++++++++++++++++++++++++++++
 target/s390x/kvm/kvm_s390x.h |  2 ++
 target/s390x/sigp.c          | 19 +++++++++++++++++-
 6 files changed, 82 insertions(+), 1 deletion(-)

diff --git a/target/s390x/cpu-sysemu.c b/target/s390x/cpu-sysemu.c
index 5471e01ee8..a39dad63d4 100644
--- a/target/s390x/cpu-sysemu.c
+++ b/target/s390x/cpu-sysemu.c
@@ -254,6 +254,21 @@ unsigned int s390_cpu_set_state(uint8_t cpu_state, S390CPU 
*cpu)
     return s390_count_running_cpus();
 }
 
+int s390_cpu_set_busy(S390CPU *cpu, int order)
+{
+    if (kvm_enabled()) {
+        return kvm_s390_vcpu_set_busy(cpu, order);
+    }
+    return 0;
+}
+
+void s390_cpu_reset_busy(S390CPU *cpu)
+{
+    if (kvm_enabled()) {
+        kvm_s390_vcpu_reset_busy(cpu);
+    }
+}
+
 int s390_set_memory_limit(uint64_t new_limit, uint64_t *hw_limit)
 {
     if (kvm_enabled()) {
diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
index 7b7b05f1d3..4388a7fdee 100644
--- a/target/s390x/cpu.c
+++ b/target/s390x/cpu.c
@@ -115,6 +115,7 @@ static void s390_cpu_reset(CPUState *s, cpu_reset_type type)
 
     scc->parent_reset(dev);
     cpu->env.sigp_order = 0;
+    s390_cpu_reset_busy(cpu);
     s390_cpu_set_state(S390_CPU_STATE_STOPPED, cpu);
 
     switch (type) {
diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h
index ca3845d023..890a912a91 100644
--- a/target/s390x/cpu.h
+++ b/target/s390x/cpu.h
@@ -780,11 +780,19 @@ int s390_assign_subch_ioeventfd(EventNotifier *notifier, 
uint32_t sch_id,
                                 int vq, bool assign);
 #ifndef CONFIG_USER_ONLY
 unsigned int s390_cpu_set_state(uint8_t cpu_state, S390CPU *cpu);
+int s390_cpu_set_busy(S390CPU *cpu, int order);
+void s390_cpu_reset_busy(S390CPU *cpu);
 #else
 static inline unsigned int s390_cpu_set_state(uint8_t cpu_state, S390CPU *cpu)
 {
     return 0;
 }
+static inline int s390_cpu_set_busy(S390CPU *cpu, int order)
+{
+}
+static inline void s390_cpu_reset_busy(S390CPU *cpu)
+{
+}
 #endif /* CONFIG_USER_ONLY */
 static inline uint8_t s390_cpu_get_state(S390CPU *cpu)
 {
diff --git a/target/s390x/kvm/kvm.c b/target/s390x/kvm/kvm.c
index 5b1fdb55c4..9907328caa 100644
--- a/target/s390x/kvm/kvm.c
+++ b/target/s390x/kvm/kvm.c
@@ -157,6 +157,7 @@ static int cap_ri;
 static int cap_hpage_1m;
 static int cap_vcpu_resets;
 static int cap_protected;
+static int cap_user_busy;
 
 static int active_cmma;
 
@@ -358,6 +359,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
     cap_vcpu_resets = kvm_check_extension(s, KVM_CAP_S390_VCPU_RESETS);
     cap_protected = kvm_check_extension(s, KVM_CAP_S390_PROTECTED);
+    cap_user_busy = kvm_check_extension(s, KVM_CAP_S390_USER_BUSY);
 
     kvm_vm_enable_cap(s, KVM_CAP_S390_USER_SIGP, 0);
     kvm_vm_enable_cap(s, KVM_CAP_S390_VECTOR_REGISTERS, 0);
@@ -382,6 +384,10 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
         kvm_vm_enable_cap(s, KVM_CAP_S390_AIS, 0);
     }
 
+    if (cap_user_busy) {
+        kvm_vm_enable_cap(s, KVM_CAP_S390_USER_BUSY, 0);
+    }
+
     kvm_set_max_memslot_size(KVM_SLOT_MAX_BYTES);
     return 0;
 }
@@ -2558,6 +2564,38 @@ void kvm_s390_stop_interrupt(S390CPU *cpu)
     kvm_s390_vcpu_interrupt(cpu, &irq);
 }
 
+int kvm_s390_vcpu_set_busy(S390CPU *cpu, unsigned int order)
+{
+    CPUState *cs = CPU(cpu);
+    struct kvm_s390_user_busy_info busy = {
+        .reason = KVM_S390_USER_BUSY_REASON_SIGP,
+        .function = KVM_S390_USER_BUSY_FUNCTION_SET,
+        .payload = order,
+    };
+
+    if (!cap_user_busy) {
+        return 0;
+    }
+
+    return kvm_vcpu_ioctl(cs, KVM_S390_USER_BUSY, &busy);
+}
+
+void kvm_s390_vcpu_reset_busy(S390CPU *cpu)
+{
+    CPUState *cs = CPU(cpu);
+    struct kvm_s390_user_busy_info busy = {
+        .reason = KVM_S390_USER_BUSY_REASON_SIGP,
+        .function = KVM_S390_USER_BUSY_FUNCTION_RESET,
+    };
+
+    if (!cap_user_busy) {
+        return;
+    }
+
+    /* Don't care about the response from this */
+    kvm_vcpu_ioctl(cs, KVM_S390_USER_BUSY, &busy);
+}
+
 bool kvm_arch_cpu_check_are_resettable(void)
 {
     return true;
diff --git a/target/s390x/kvm/kvm_s390x.h b/target/s390x/kvm/kvm_s390x.h
index 05a5e1e6f4..2f9f761c2b 100644
--- a/target/s390x/kvm/kvm_s390x.h
+++ b/target/s390x/kvm/kvm_s390x.h
@@ -45,5 +45,7 @@ void kvm_s390_crypto_reset(void);
 void kvm_s390_restart_interrupt(S390CPU *cpu);
 void kvm_s390_stop_interrupt(S390CPU *cpu);
 void kvm_s390_set_diag318(CPUState *cs, uint64_t diag318_info);
+int kvm_s390_vcpu_set_busy(S390CPU *cpu, unsigned int order);
+void kvm_s390_vcpu_reset_busy(S390CPU *cpu);
 
 #endif /* KVM_S390X_H */
diff --git a/target/s390x/sigp.c b/target/s390x/sigp.c
index 51c727834c..b54a49f8e5 100644
--- a/target/s390x/sigp.c
+++ b/target/s390x/sigp.c
@@ -111,12 +111,14 @@ static void sigp_stop(CPUState *cs, run_on_cpu_data arg)
 
     if (s390_cpu_get_state(cpu) != S390_CPU_STATE_OPERATING) {
         si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
+        s390_cpu_reset_busy(cpu);
         return;
     }
 
     /* disabled wait - sleeping in user space */
     if (cs->halted) {
         s390_cpu_set_state(S390_CPU_STATE_STOPPED, cpu);
+        s390_cpu_reset_busy(cpu);
     } else {
         /* execute the stop function */
         cpu->env.sigp_order = SIGP_STOP;
@@ -139,12 +141,13 @@ static void sigp_stop_and_store_status(CPUState *cs, 
run_on_cpu_data arg)
     case S390_CPU_STATE_OPERATING:
         cpu->env.sigp_order = SIGP_STOP_STORE_STATUS;
         cpu_inject_stop(cpu);
-        /* store will be performed in do_stop_interrup() */
+        /* store will be performed in do_stop_interrupt() */
         break;
     case S390_CPU_STATE_STOPPED:
         /* already stopped, just store the status */
         cpu_synchronize_state(cs);
         s390_store_status(cpu, S390_STORE_STATUS_DEF_ADDR, true);
+        s390_cpu_reset_busy(cpu);
         break;
     }
     si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
@@ -375,6 +378,10 @@ static int handle_sigp_single_dst(S390CPU *cpu, S390CPU 
*dst_cpu, uint8_t order,
         return SIGP_CC_BUSY;
     }
 
+    if (s390_cpu_set_busy(dst_cpu, order) == -EBUSY) {
+        return SIGP_CC_BUSY;
+    }
+
     switch (order) {
     case SIGP_SENSE:
         sigp_sense(dst_cpu, &si);
@@ -422,6 +429,15 @@ static int handle_sigp_single_dst(S390CPU *cpu, S390CPU 
*dst_cpu, uint8_t order,
         set_sigp_status(&si, SIGP_STAT_INVALID_ORDER);
     }
 
+    switch (order) {
+    case SIGP_STOP:
+    case SIGP_STOP_STORE_STATUS:
+        /* These orders will clean up the indicator when they are finished */
+        break;
+    default:
+        s390_cpu_reset_busy(dst_cpu);
+    }
+
     return si.cc;
 }
 
@@ -487,6 +503,7 @@ void do_stop_interrupt(CPUS390XState *env)
     }
     env->sigp_order = 0;
     env->pending_int &= ~INTERRUPT_STOP;
+    s390_cpu_reset_busy(cpu);
 }
 
 void s390_init_sigp(void)
-- 
2.25.1