[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PATCH v9 16/20] multi-process: create IOHUB object to handle irq
From: |
elena . ufimtseva |
Subject: |
[PATCH v9 16/20] multi-process: create IOHUB object to handle irq |
Date: |
Thu, 27 Aug 2020 11:12:27 -0700 |
From: Jagannathan Raman <jag.raman@oracle.com>
IOHUB object is added to manage PCI IRQs. It uses KVM_IRQFD
ioctl to create irqfd to injecting PCI interrupts to the guest.
IOHUB object forwards the irqfd to the remote process. Remote process
uses this fd to directly send interrupts to the guest, bypassing QEMU.
Signed-off-by: John G Johnson <john.g.johnson@oracle.com>
Signed-off-by: Jagannathan Raman <jag.raman@oracle.com>
Signed-off-by: Elena Ufimtseva <elena.ufimtseva@oracle.com>
---
MAINTAINERS | 2 +
hw/i386/meson.build | 1 +
hw/i386/remote-iohub.c | 123 +++++++++++++++++++++++++++++++++
hw/i386/remote-msg.c | 4 ++
hw/i386/remote.c | 10 +++
hw/pci/proxy.c | 58 ++++++++++++++++
include/hw/i386/remote-iohub.h | 42 +++++++++++
include/hw/i386/remote.h | 3 +
include/hw/pci/pci_ids.h | 3 +
include/hw/pci/proxy.h | 5 ++
include/io/mpqemu-link.h | 1 +
io/mpqemu-link.c | 5 ++
12 files changed, 257 insertions(+)
create mode 100644 hw/i386/remote-iohub.c
create mode 100644 include/hw/i386/remote-iohub.h
diff --git a/MAINTAINERS b/MAINTAINERS
index aa18f4fe86..084a82232a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -3056,6 +3056,8 @@ F: hw/pci/proxy.c
F: include/hw/pci/proxy.h
F: hw/pci/memory-sync.c
F: include/hw/pci/memory-sync.h
+F: hw/i386/remote-iohub.c
+F: include/hw/i386/remote-iohub.h
Build and test automation
-------------------------
diff --git a/hw/i386/meson.build b/hw/i386/meson.build
index 36e151c80a..1adfce948a 100644
--- a/hw/i386/meson.build
+++ b/hw/i386/meson.build
@@ -27,6 +27,7 @@ i386_ss.add(when: 'CONFIG_MPQEMU', if_true: files('remote.c'))
i386_ss.add(when: 'CONFIG_MPQEMU', if_true: files('remote-msg.c'))
i386_ss.add(when: 'CONFIG_MPQEMU', if_true: files('remote-obj.c'))
i386_ss.add(when: 'CONFIG_MPQEMU', if_true: files('remote-memory.c'))
+i386_ss.add(when: 'CONFIG_MPQEMU', if_true: files('remote-iohub.c'))
subdir('kvm')
subdir('xen')
diff --git a/hw/i386/remote-iohub.c b/hw/i386/remote-iohub.c
new file mode 100644
index 0000000000..e59424ea32
--- /dev/null
+++ b/hw/i386/remote-iohub.c
@@ -0,0 +1,123 @@
+/*
+ * Remote IO Hub
+ *
+ * Copyright © 2018, 2020 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_ids.h"
+#include "hw/pci/pci_bus.h"
+#include "qemu/thread.h"
+#include "hw/boards.h"
+#include "hw/i386/remote.h"
+#include "hw/i386/remote-iohub.h"
+#include "qemu/main-loop.h"
+
+void remote_iohub_init(RemoteIOHubState *iohub)
+{
+ int pirq;
+
+ memset(&iohub->irqfds, 0, sizeof(iohub->irqfds));
+ memset(&iohub->resamplefds, 0, sizeof(iohub->resamplefds));
+
+ for (pirq = 0; pirq < REMOTE_IOHUB_NB_PIRQS; pirq++) {
+ qemu_mutex_init(&iohub->irq_level_lock[pirq]);
+ iohub->irq_level[pirq] = 0;
+ event_notifier_init_fd(&iohub->irqfds[pirq], -1);
+ event_notifier_init_fd(&iohub->resamplefds[pirq], -1);
+ }
+}
+
+void remote_iohub_finalize(RemoteIOHubState *iohub)
+{
+ int pirq;
+
+ for (pirq = 0; pirq < REMOTE_IOHUB_NB_PIRQS; pirq++) {
+ qemu_set_fd_handler(event_notifier_get_fd(&iohub->resamplefds[pirq]),
+ NULL, NULL, NULL);
+ event_notifier_cleanup(&iohub->irqfds[pirq]);
+ event_notifier_cleanup(&iohub->resamplefds[pirq]);
+ qemu_mutex_destroy(&iohub->irq_level_lock[pirq]);
+ }
+}
+
+int remote_iohub_map_irq(PCIDevice *pci_dev, int intx)
+{
+ return pci_dev->devfn;
+}
+
+void remote_iohub_set_irq(void *opaque, int pirq, int level)
+{
+ RemoteIOHubState *iohub = opaque;
+
+ assert(pirq >= 0);
+ assert(pirq < PCI_DEVFN_MAX);
+
+ qemu_mutex_lock(&iohub->irq_level_lock[pirq]);
+
+ if (level) {
+ if (++iohub->irq_level[pirq] == 1) {
+ event_notifier_set(&iohub->irqfds[pirq]);
+ }
+ } else if (iohub->irq_level[pirq] > 0) {
+ iohub->irq_level[pirq]--;
+ }
+
+ qemu_mutex_unlock(&iohub->irq_level_lock[pirq]);
+}
+
+static void intr_resample_handler(void *opaque)
+{
+ ResampleToken *token = opaque;
+ RemoteIOHubState *iohub = token->iohub;
+ int pirq, s;
+
+ pirq = token->pirq;
+
+ s = event_notifier_test_and_clear(&iohub->resamplefds[pirq]);
+
+ assert(s >= 0);
+
+ qemu_mutex_lock(&iohub->irq_level_lock[pirq]);
+
+ if (iohub->irq_level[pirq]) {
+ event_notifier_set(&iohub->irqfds[pirq]);
+ }
+
+ qemu_mutex_unlock(&iohub->irq_level_lock[pirq]);
+}
+
+void process_set_irqfd_msg(PCIDevice *pci_dev, MPQemuMsg *msg)
+{
+ RemoteMachineState *machine = REMOTE_MACHINE(current_machine);
+ RemoteIOHubState *iohub = &machine->iohub;
+ int pirq, intx;
+
+ intx = pci_get_byte(pci_dev->config + PCI_INTERRUPT_PIN) - 1;
+
+ pirq = remote_iohub_map_irq(pci_dev, intx);
+
+ if (event_notifier_get_fd(&iohub->irqfds[pirq]) != -1) {
+ qemu_set_fd_handler(event_notifier_get_fd(&iohub->resamplefds[pirq]),
+ NULL, NULL, NULL);
+ event_notifier_cleanup(&iohub->irqfds[pirq]);
+ event_notifier_cleanup(&iohub->resamplefds[pirq]);
+ memset(&iohub->token[pirq], 0, sizeof(ResampleToken));
+ }
+
+ event_notifier_init_fd(&iohub->irqfds[pirq], msg->fds[0]);
+ event_notifier_init_fd(&iohub->resamplefds[pirq], msg->fds[1]);
+
+ iohub->token[pirq].iohub = iohub;
+ iohub->token[pirq].pirq = pirq;
+
+ qemu_set_fd_handler(msg->fds[1], intr_resample_handler, NULL,
+ &iohub->token[pirq]);
+}
diff --git a/hw/i386/remote-msg.c b/hw/i386/remote-msg.c
index 017224505b..2707fe81c4 100644
--- a/hw/i386/remote-msg.c
+++ b/hw/i386/remote-msg.c
@@ -18,6 +18,7 @@
#include "hw/pci/pci.h"
#include "exec/memattrs.h"
#include "hw/i386/remote-memory.h"
+#include "hw/i386/remote-iohub.h"
static void process_config_write(QIOChannel *ioc, PCIDevice *dev,
MPQemuMsg *msg);
@@ -75,6 +76,9 @@ gboolean mpqemu_process_msg(QIOChannel *ioc, GIOCondition
cond,
case SYNC_SYSMEM:
remote_sysmem_reconfig(&msg, &local_err);
break;
+ case SET_IRQFD:
+ process_set_irqfd_msg(pci_dev, &msg);
+ break;
default:
error_setg(&local_err,
"Unknown command (%d) received for device %s (pid=%d)",
diff --git a/hw/i386/remote.c b/hw/i386/remote.c
index a67be33396..95de652e79 100644
--- a/hw/i386/remote.c
+++ b/hw/i386/remote.c
@@ -20,12 +20,15 @@
#include "exec/address-spaces.h"
#include "exec/memory.h"
#include "qapi/error.h"
+#include "hw/pci/pci_host.h"
+#include "hw/i386/remote-iohub.h"
static void remote_machine_init(MachineState *machine)
{
MemoryRegion *system_memory, *system_io, *pci_memory;
RemoteMachineState *s = REMOTE_MACHINE(machine);
RemotePCIHost *rem_host;
+ PCIHostState *pci_host;
system_memory = get_system_memory();
system_io = get_system_io();
@@ -45,6 +48,13 @@ static void remote_machine_init(MachineState *machine)
memory_region_add_subregion_overlap(system_memory, 0x0, pci_memory, -1);
qdev_realize(DEVICE(rem_host), sysbus_get_default(), &error_fatal);
+
+ pci_host = PCI_HOST_BRIDGE(rem_host);
+
+ remote_iohub_init(&s->iohub);
+
+ pci_bus_irqs(pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq,
+ &s->iohub, REMOTE_IOHUB_NB_PIRQS);
}
static void remote_machine_class_init(ObjectClass *oc, void *data)
diff --git a/hw/pci/proxy.c b/hw/pci/proxy.c
index 3e13d7ea5e..4a1aab884a 100644
--- a/hw/pci/proxy.c
+++ b/hw/pci/proxy.c
@@ -19,6 +19,9 @@
#include "qemu/error-report.h"
#include "hw/pci/memory-sync.h"
#include "qom/object.h"
+#include "qemu/event_notifier.h"
+#include "sysemu/kvm.h"
+#include "util/event_notifier-posix.c"
static void proxy_set_socket(PCIProxyDev *pdev, int fd, Error **errp)
{
@@ -30,6 +33,56 @@ static Property proxy_properties[] = {
DEFINE_PROP_END_OF_LIST(),
};
+static void proxy_intx_update(PCIDevice *pci_dev)
+{
+ PCIProxyDev *dev = PCI_PROXY_DEV(pci_dev);
+ PCIINTxRoute route;
+ int pin = pci_get_byte(pci_dev->config + PCI_INTERRUPT_PIN) - 1;
+
+ if (dev->virq != -1) {
+ kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &dev->intr,
dev->virq);
+ dev->virq = -1;
+ }
+
+ route = pci_device_route_intx_to_irq(pci_dev, pin);
+
+ dev->virq = route.irq;
+
+ if (dev->virq != -1) {
+ kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &dev->intr,
+ &dev->resample, dev->virq);
+ }
+}
+
+static void setup_irqfd(PCIProxyDev *dev)
+{
+ PCIDevice *pci_dev = PCI_DEVICE(dev);
+ MPQemuMsg msg;
+ Error *local_err = NULL;
+
+ event_notifier_init(&dev->intr, 0);
+ event_notifier_init(&dev->resample, 0);
+
+ memset(&msg, 0, sizeof(MPQemuMsg));
+ msg.cmd = SET_IRQFD;
+ msg.num_fds = 2;
+ msg.fds[0] = event_notifier_get_fd(&dev->intr);
+ msg.fds[1] = event_notifier_get_fd(&dev->resample);
+ msg.size = 0;
+
+ mpqemu_msg_send(&msg, dev->ioc, &local_err);
+ if (local_err) {
+ error_report("Error to send cmd to remote process %d",
+ msg.cmd);
+ }
+
+ dev->virq = -1;
+
+ proxy_intx_update(pci_dev);
+
+ pci_device_set_intx_routing_notifier(pci_dev, proxy_intx_update);
+}
+
static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)
{
PCIProxyDev *dev = PCI_PROXY_DEV(device);
@@ -52,6 +105,8 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error
**errp)
qio_channel_set_blocking(dev->ioc, true, NULL);
configure_memory_sync(&dev->sync, dev->ioc);
+
+ setup_irqfd(dev);
}
static void pci_proxy_dev_exit(PCIDevice *pdev)
@@ -61,6 +116,9 @@ static void pci_proxy_dev_exit(PCIDevice *pdev)
qio_channel_close(dev->ioc, NULL);
deconfigure_memory_sync(&dev->sync);
+
+ event_notifier_cleanup(&dev->intr);
+ event_notifier_cleanup(&dev->resample);
}
static int config_op_send(PCIProxyDev *pdev, uint32_t addr, uint32_t *val,
diff --git a/include/hw/i386/remote-iohub.h b/include/hw/i386/remote-iohub.h
new file mode 100644
index 0000000000..cd59476cb7
--- /dev/null
+++ b/include/hw/i386/remote-iohub.h
@@ -0,0 +1,42 @@
+/*
+ * IO Hub for remote device
+ *
+ * Copyright © 2018, 2020 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef REMOTE_IOHUB_H
+#define REMOTE_IOHUB_H
+
+#include "hw/pci/pci.h"
+#include "qemu/event_notifier.h"
+#include "qemu/thread-posix.h"
+#include "io/mpqemu-link.h"
+
+#define REMOTE_IOHUB_NB_PIRQS PCI_DEVFN_MAX
+
+typedef struct ResampleToken {
+ void *iohub;
+ int pirq;
+} ResampleToken;
+
+typedef struct RemoteIOHubState {
+ PCIDevice d;
+ EventNotifier irqfds[REMOTE_IOHUB_NB_PIRQS];
+ EventNotifier resamplefds[REMOTE_IOHUB_NB_PIRQS];
+ unsigned int irq_level[REMOTE_IOHUB_NB_PIRQS];
+ ResampleToken token[REMOTE_IOHUB_NB_PIRQS];
+ QemuMutex irq_level_lock[REMOTE_IOHUB_NB_PIRQS];
+} RemoteIOHubState;
+
+int remote_iohub_map_irq(PCIDevice *pci_dev, int intx);
+void remote_iohub_set_irq(void *opaque, int pirq, int level);
+void process_set_irqfd_msg(PCIDevice *pci_dev, MPQemuMsg *msg);
+
+void remote_iohub_init(RemoteIOHubState *iohub);
+void remote_iohub_finalize(RemoteIOHubState *iohub);
+
+#endif
diff --git a/include/hw/i386/remote.h b/include/hw/i386/remote.h
index 7f9028fcce..a1ebfb6457 100644
--- a/include/hw/i386/remote.h
+++ b/include/hw/i386/remote.h
@@ -15,11 +15,14 @@
#include "hw/boards.h"
#include "hw/pci-host/remote.h"
#include "io/channel.h"
+#include "hw/i386/remote-iohub.h"
typedef struct RemoteMachineState {
MachineState parent_obj;
RemotePCIHost *host;
+
+ RemoteIOHubState iohub;
} RemoteMachineState;
#define TYPE_REMOTE_MACHINE "remote-machine"
diff --git a/include/hw/pci/pci_ids.h b/include/hw/pci/pci_ids.h
index 11f8ab7149..bd0c17dc78 100644
--- a/include/hw/pci/pci_ids.h
+++ b/include/hw/pci/pci_ids.h
@@ -192,6 +192,9 @@
#define PCI_DEVICE_ID_SUN_SIMBA 0x5000
#define PCI_DEVICE_ID_SUN_SABRE 0xa000
+#define PCI_VENDOR_ID_ORACLE 0x108e
+#define PCI_DEVICE_ID_REMOTE_IOHUB 0xb000
+
#define PCI_VENDOR_ID_CMD 0x1095
#define PCI_DEVICE_ID_CMD_646 0x0646
diff --git a/include/hw/pci/proxy.h b/include/hw/pci/proxy.h
index e3583dc6fe..63660aa64f 100644
--- a/include/hw/pci/proxy.h
+++ b/include/hw/pci/proxy.h
@@ -12,6 +12,7 @@
#include "hw/pci/pci.h"
#include "io/channel.h"
#include "hw/pci/memory-sync.h"
+#include "qemu/event_notifier.h"
#define TYPE_PCI_PROXY_DEV "pci-proxy-dev"
@@ -43,6 +44,10 @@ struct PCIProxyDev {
RemoteMemSync sync;
+ int virq;
+ EventNotifier intr;
+ EventNotifier resample;
+
ProxyMemoryRegion region[PCI_NUM_REGIONS];
};
diff --git a/include/io/mpqemu-link.h b/include/io/mpqemu-link.h
index 967ae24e3a..abcd89a588 100644
--- a/include/io/mpqemu-link.h
+++ b/include/io/mpqemu-link.h
@@ -38,6 +38,7 @@ typedef enum {
PCI_CONFIG_READ,
BAR_WRITE,
BAR_READ,
+ SET_IRQFD,
MAX = INT_MAX,
} MPQemuCmd;
diff --git a/io/mpqemu-link.c b/io/mpqemu-link.c
index 9233df215e..7acdc236be 100644
--- a/io/mpqemu-link.c
+++ b/io/mpqemu-link.c
@@ -334,6 +334,11 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)
return false;
}
break;
+ case SET_IRQFD:
+ if (msg->size || (msg->num_fds != 2)) {
+ return false;
+ }
+ break;
default:
break;
}
--
2.25.GIT
- [PATCH v9 05/20] multi-process: add qio channel function to transmit, (continued)
- [PATCH v9 05/20] multi-process: add qio channel function to transmit, elena . ufimtseva, 2020/08/27
- [PATCH v9 06/20] multi-process: define MPQemuMsg format and transmission functions, elena . ufimtseva, 2020/08/27
- [PATCH v9 09/20] multi-process: Associate fd of a PCIDevice with its object, elena . ufimtseva, 2020/08/27
- [PATCH v9 10/20] multi-process: setup memory manager for remote device, elena . ufimtseva, 2020/08/27
- [PATCH v9 12/20] multi-process: add proxy communication functions, elena . ufimtseva, 2020/08/27
- [PATCH v9 14/20] multi-process: PCI BAR read/write handling for proxy & remote endpoints, elena . ufimtseva, 2020/08/27
- [PATCH v9 13/20] multi-process: Forward PCI config space acceses to the remote process, elena . ufimtseva, 2020/08/27
- [PATCH v9 15/20] multi-process: Synchronize remote memory, elena . ufimtseva, 2020/08/27
- [PATCH v9 17/20] multi-process: Retrieve PCI info from remote process, elena . ufimtseva, 2020/08/27
- [PATCH v9 18/20] multi-process: perform device reset in the remote process, elena . ufimtseva, 2020/08/27
- [PATCH v9 16/20] multi-process: create IOHUB object to handle irq,
elena . ufimtseva <=
- [PATCH v9 20/20] multi-process: add configure and usage information, elena . ufimtseva, 2020/08/27
- [PATCH v9 19/20] multi-process: add the concept description to docs/devel/qemu-multiprocess, elena . ufimtseva, 2020/08/27
- [PATCH v9 01/20] memory: alloc RAM from file at offset, elena . ufimtseva, 2020/08/27
- [PATCH v9 07/20] multi-process: define transmission functions in remote, elena . ufimtseva, 2020/08/27
- [PATCH v9 08/20] multi-process: Initialize message handler in remote device, elena . ufimtseva, 2020/08/27
- [PATCH v9 11/20] multi-process: introduce proxy object, elena . ufimtseva, 2020/08/27