[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PATCH v10 15/19] multi-process: create IOHUB object to handle irq
From: |
elena . ufimtseva |
Subject: |
[PATCH v10 15/19] multi-process: create IOHUB object to handle irq |
Date: |
Mon, 5 Oct 2020 11:51:03 -0700 |
From: Jagannathan Raman <jag.raman@oracle.com>
IOHUB object is added to manage PCI IRQs. It uses KVM_IRQFD
ioctl to create irqfd to injecting PCI interrupts to the guest.
IOHUB object forwards the irqfd to the remote process. Remote process
uses this fd to directly send interrupts to the guest, bypassing QEMU.
Signed-off-by: John G Johnson <john.g.johnson@oracle.com>
Signed-off-by: Jagannathan Raman <jag.raman@oracle.com>
Signed-off-by: Elena Ufimtseva <elena.ufimtseva@oracle.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
---
MAINTAINERS | 2 +
hw/i386/meson.build | 1 +
hw/i386/remote-iohub.c | 123 +++++++++++++++++++++++++++++++++
hw/i386/remote-msg.c | 4 ++
hw/i386/remote.c | 10 +++
hw/pci/proxy.c | 58 ++++++++++++++++
include/hw/i386/remote-iohub.h | 42 +++++++++++
include/hw/i386/remote.h | 3 +
include/hw/pci/pci_ids.h | 3 +
include/hw/pci/proxy.h | 5 ++
include/io/mpqemu-link.h | 1 +
io/mpqemu-link.c | 5 ++
12 files changed, 257 insertions(+)
create mode 100644 hw/i386/remote-iohub.c
create mode 100644 include/hw/i386/remote-iohub.h
diff --git a/MAINTAINERS b/MAINTAINERS
index aa18f4fe86..084a82232a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -3056,6 +3056,8 @@ F: hw/pci/proxy.c
F: include/hw/pci/proxy.h
F: hw/pci/memory-sync.c
F: include/hw/pci/memory-sync.h
+F: hw/i386/remote-iohub.c
+F: include/hw/i386/remote-iohub.h
Build and test automation
-------------------------
diff --git a/hw/i386/meson.build b/hw/i386/meson.build
index 36e151c80a..1adfce948a 100644
--- a/hw/i386/meson.build
+++ b/hw/i386/meson.build
@@ -27,6 +27,7 @@ i386_ss.add(when: 'CONFIG_MPQEMU', if_true: files('remote.c'))
i386_ss.add(when: 'CONFIG_MPQEMU', if_true: files('remote-msg.c'))
i386_ss.add(when: 'CONFIG_MPQEMU', if_true: files('remote-obj.c'))
i386_ss.add(when: 'CONFIG_MPQEMU', if_true: files('remote-memory.c'))
+i386_ss.add(when: 'CONFIG_MPQEMU', if_true: files('remote-iohub.c'))
subdir('kvm')
subdir('xen')
diff --git a/hw/i386/remote-iohub.c b/hw/i386/remote-iohub.c
new file mode 100644
index 0000000000..e59424ea32
--- /dev/null
+++ b/hw/i386/remote-iohub.c
@@ -0,0 +1,123 @@
+/*
+ * Remote IO Hub
+ *
+ * Copyright © 2018, 2020 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_ids.h"
+#include "hw/pci/pci_bus.h"
+#include "qemu/thread.h"
+#include "hw/boards.h"
+#include "hw/i386/remote.h"
+#include "hw/i386/remote-iohub.h"
+#include "qemu/main-loop.h"
+
+void remote_iohub_init(RemoteIOHubState *iohub)
+{
+ int pirq;
+
+ memset(&iohub->irqfds, 0, sizeof(iohub->irqfds));
+ memset(&iohub->resamplefds, 0, sizeof(iohub->resamplefds));
+
+ for (pirq = 0; pirq < REMOTE_IOHUB_NB_PIRQS; pirq++) {
+ qemu_mutex_init(&iohub->irq_level_lock[pirq]);
+ iohub->irq_level[pirq] = 0;
+ event_notifier_init_fd(&iohub->irqfds[pirq], -1);
+ event_notifier_init_fd(&iohub->resamplefds[pirq], -1);
+ }
+}
+
+void remote_iohub_finalize(RemoteIOHubState *iohub)
+{
+ int pirq;
+
+ for (pirq = 0; pirq < REMOTE_IOHUB_NB_PIRQS; pirq++) {
+ qemu_set_fd_handler(event_notifier_get_fd(&iohub->resamplefds[pirq]),
+ NULL, NULL, NULL);
+ event_notifier_cleanup(&iohub->irqfds[pirq]);
+ event_notifier_cleanup(&iohub->resamplefds[pirq]);
+ qemu_mutex_destroy(&iohub->irq_level_lock[pirq]);
+ }
+}
+
+int remote_iohub_map_irq(PCIDevice *pci_dev, int intx)
+{
+ return pci_dev->devfn;
+}
+
+void remote_iohub_set_irq(void *opaque, int pirq, int level)
+{
+ RemoteIOHubState *iohub = opaque;
+
+ assert(pirq >= 0);
+ assert(pirq < PCI_DEVFN_MAX);
+
+ qemu_mutex_lock(&iohub->irq_level_lock[pirq]);
+
+ if (level) {
+ if (++iohub->irq_level[pirq] == 1) {
+ event_notifier_set(&iohub->irqfds[pirq]);
+ }
+ } else if (iohub->irq_level[pirq] > 0) {
+ iohub->irq_level[pirq]--;
+ }
+
+ qemu_mutex_unlock(&iohub->irq_level_lock[pirq]);
+}
+
+static void intr_resample_handler(void *opaque)
+{
+ ResampleToken *token = opaque;
+ RemoteIOHubState *iohub = token->iohub;
+ int pirq, s;
+
+ pirq = token->pirq;
+
+ s = event_notifier_test_and_clear(&iohub->resamplefds[pirq]);
+
+ assert(s >= 0);
+
+ qemu_mutex_lock(&iohub->irq_level_lock[pirq]);
+
+ if (iohub->irq_level[pirq]) {
+ event_notifier_set(&iohub->irqfds[pirq]);
+ }
+
+ qemu_mutex_unlock(&iohub->irq_level_lock[pirq]);
+}
+
+void process_set_irqfd_msg(PCIDevice *pci_dev, MPQemuMsg *msg)
+{
+ RemoteMachineState *machine = REMOTE_MACHINE(current_machine);
+ RemoteIOHubState *iohub = &machine->iohub;
+ int pirq, intx;
+
+ intx = pci_get_byte(pci_dev->config + PCI_INTERRUPT_PIN) - 1;
+
+ pirq = remote_iohub_map_irq(pci_dev, intx);
+
+ if (event_notifier_get_fd(&iohub->irqfds[pirq]) != -1) {
+ qemu_set_fd_handler(event_notifier_get_fd(&iohub->resamplefds[pirq]),
+ NULL, NULL, NULL);
+ event_notifier_cleanup(&iohub->irqfds[pirq]);
+ event_notifier_cleanup(&iohub->resamplefds[pirq]);
+ memset(&iohub->token[pirq], 0, sizeof(ResampleToken));
+ }
+
+ event_notifier_init_fd(&iohub->irqfds[pirq], msg->fds[0]);
+ event_notifier_init_fd(&iohub->resamplefds[pirq], msg->fds[1]);
+
+ iohub->token[pirq].iohub = iohub;
+ iohub->token[pirq].pirq = pirq;
+
+ qemu_set_fd_handler(msg->fds[1], intr_resample_handler, NULL,
+ &iohub->token[pirq]);
+}
diff --git a/hw/i386/remote-msg.c b/hw/i386/remote-msg.c
index 3465a9ddd6..a318a9ff91 100644
--- a/hw/i386/remote-msg.c
+++ b/hw/i386/remote-msg.c
@@ -18,6 +18,7 @@
#include "hw/pci/pci.h"
#include "exec/memattrs.h"
#include "hw/i386/remote-memory.h"
+#include "hw/i386/remote-iohub.h"
static void process_config_write(QIOChannel *ioc, PCIDevice *dev,
MPQemuMsg *msg);
@@ -68,6 +69,9 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)
case SYNC_SYSMEM:
remote_sysmem_reconfig(&msg, &local_err);
break;
+ case SET_IRQFD:
+ process_set_irqfd_msg(pci_dev, &msg);
+ break;
default:
error_setg(&local_err,
diff --git a/hw/i386/remote.c b/hw/i386/remote.c
index a67be33396..95de652e79 100644
--- a/hw/i386/remote.c
+++ b/hw/i386/remote.c
@@ -20,12 +20,15 @@
#include "exec/address-spaces.h"
#include "exec/memory.h"
#include "qapi/error.h"
+#include "hw/pci/pci_host.h"
+#include "hw/i386/remote-iohub.h"
static void remote_machine_init(MachineState *machine)
{
MemoryRegion *system_memory, *system_io, *pci_memory;
RemoteMachineState *s = REMOTE_MACHINE(machine);
RemotePCIHost *rem_host;
+ PCIHostState *pci_host;
system_memory = get_system_memory();
system_io = get_system_io();
@@ -45,6 +48,13 @@ static void remote_machine_init(MachineState *machine)
memory_region_add_subregion_overlap(system_memory, 0x0, pci_memory, -1);
qdev_realize(DEVICE(rem_host), sysbus_get_default(), &error_fatal);
+
+ pci_host = PCI_HOST_BRIDGE(rem_host);
+
+ remote_iohub_init(&s->iohub);
+
+ pci_bus_irqs(pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq,
+ &s->iohub, REMOTE_IOHUB_NB_PIRQS);
}
static void remote_machine_class_init(ObjectClass *oc, void *data)
diff --git a/hw/pci/proxy.c b/hw/pci/proxy.c
index 008af7e8f5..029e78e85f 100644
--- a/hw/pci/proxy.c
+++ b/hw/pci/proxy.c
@@ -20,6 +20,9 @@
#include "qemu/error-report.h"
#include "hw/pci/memory-sync.h"
#include "qom/object.h"
+#include "qemu/event_notifier.h"
+#include "sysemu/kvm.h"
+#include "util/event_notifier-posix.c"
static void proxy_set_socket(PCIProxyDev *pdev, int fd, Error **errp)
{
@@ -31,6 +34,56 @@ static Property proxy_properties[] = {
DEFINE_PROP_END_OF_LIST(),
};
+static void proxy_intx_update(PCIDevice *pci_dev)
+{
+ PCIProxyDev *dev = PCI_PROXY_DEV(pci_dev);
+ PCIINTxRoute route;
+ int pin = pci_get_byte(pci_dev->config + PCI_INTERRUPT_PIN) - 1;
+
+ if (dev->virq != -1) {
+ kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &dev->intr,
dev->virq);
+ dev->virq = -1;
+ }
+
+ route = pci_device_route_intx_to_irq(pci_dev, pin);
+
+ dev->virq = route.irq;
+
+ if (dev->virq != -1) {
+ kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &dev->intr,
+ &dev->resample, dev->virq);
+ }
+}
+
+static void setup_irqfd(PCIProxyDev *dev)
+{
+ PCIDevice *pci_dev = PCI_DEVICE(dev);
+ MPQemuMsg msg;
+ Error *local_err = NULL;
+
+ event_notifier_init(&dev->intr, 0);
+ event_notifier_init(&dev->resample, 0);
+
+ memset(&msg, 0, sizeof(MPQemuMsg));
+ msg.cmd = SET_IRQFD;
+ msg.num_fds = 2;
+ msg.fds[0] = event_notifier_get_fd(&dev->intr);
+ msg.fds[1] = event_notifier_get_fd(&dev->resample);
+ msg.size = 0;
+
+ mpqemu_msg_send(&msg, dev->ioc, &local_err);
+ if (local_err) {
+ error_report("Error to send cmd to remote process %d",
+ msg.cmd);
+ }
+
+ dev->virq = -1;
+
+ proxy_intx_update(pci_dev);
+
+ pci_device_set_intx_routing_notifier(pci_dev, proxy_intx_update);
+}
+
static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)
{
PCIProxyDev *dev = PCI_PROXY_DEV(device);
@@ -62,6 +115,8 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error
**errp)
qio_channel_set_blocking(dev->ioc, true, NULL);
configure_memory_sync(&dev->sync, dev->ioc);
+
+ setup_irqfd(dev);
}
static void pci_proxy_dev_exit(PCIDevice *pdev)
@@ -71,6 +126,9 @@ static void pci_proxy_dev_exit(PCIDevice *pdev)
qio_channel_close(dev->ioc, NULL);
deconfigure_memory_sync(&dev->sync);
+
+ event_notifier_cleanup(&dev->intr);
+ event_notifier_cleanup(&dev->resample);
}
static int config_op_send(PCIProxyDev *pdev, uint32_t addr, uint32_t *val,
diff --git a/include/hw/i386/remote-iohub.h b/include/hw/i386/remote-iohub.h
new file mode 100644
index 0000000000..cd59476cb7
--- /dev/null
+++ b/include/hw/i386/remote-iohub.h
@@ -0,0 +1,42 @@
+/*
+ * IO Hub for remote device
+ *
+ * Copyright © 2018, 2020 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef REMOTE_IOHUB_H
+#define REMOTE_IOHUB_H
+
+#include "hw/pci/pci.h"
+#include "qemu/event_notifier.h"
+#include "qemu/thread-posix.h"
+#include "io/mpqemu-link.h"
+
+#define REMOTE_IOHUB_NB_PIRQS PCI_DEVFN_MAX
+
+typedef struct ResampleToken {
+ void *iohub;
+ int pirq;
+} ResampleToken;
+
+typedef struct RemoteIOHubState {
+ PCIDevice d;
+ EventNotifier irqfds[REMOTE_IOHUB_NB_PIRQS];
+ EventNotifier resamplefds[REMOTE_IOHUB_NB_PIRQS];
+ unsigned int irq_level[REMOTE_IOHUB_NB_PIRQS];
+ ResampleToken token[REMOTE_IOHUB_NB_PIRQS];
+ QemuMutex irq_level_lock[REMOTE_IOHUB_NB_PIRQS];
+} RemoteIOHubState;
+
+int remote_iohub_map_irq(PCIDevice *pci_dev, int intx);
+void remote_iohub_set_irq(void *opaque, int pirq, int level);
+void process_set_irqfd_msg(PCIDevice *pci_dev, MPQemuMsg *msg);
+
+void remote_iohub_init(RemoteIOHubState *iohub);
+void remote_iohub_finalize(RemoteIOHubState *iohub);
+
+#endif
diff --git a/include/hw/i386/remote.h b/include/hw/i386/remote.h
index e21bd5f68f..65396d48db 100644
--- a/include/hw/i386/remote.h
+++ b/include/hw/i386/remote.h
@@ -15,11 +15,14 @@
#include "hw/boards.h"
#include "hw/pci-host/remote.h"
#include "io/channel.h"
+#include "hw/i386/remote-iohub.h"
typedef struct RemoteMachineState {
MachineState parent_obj;
RemotePCIHost *host;
+
+ RemoteIOHubState iohub;
} RemoteMachineState;
/* Used to pass to co-routine device and ioc. */
diff --git a/include/hw/pci/pci_ids.h b/include/hw/pci/pci_ids.h
index 11f8ab7149..bd0c17dc78 100644
--- a/include/hw/pci/pci_ids.h
+++ b/include/hw/pci/pci_ids.h
@@ -192,6 +192,9 @@
#define PCI_DEVICE_ID_SUN_SIMBA 0x5000
#define PCI_DEVICE_ID_SUN_SABRE 0xa000
+#define PCI_VENDOR_ID_ORACLE 0x108e
+#define PCI_DEVICE_ID_REMOTE_IOHUB 0xb000
+
#define PCI_VENDOR_ID_CMD 0x1095
#define PCI_DEVICE_ID_CMD_646 0x0646
diff --git a/include/hw/pci/proxy.h b/include/hw/pci/proxy.h
index d52e2cad8a..8e582d4432 100644
--- a/include/hw/pci/proxy.h
+++ b/include/hw/pci/proxy.h
@@ -12,6 +12,7 @@
#include "hw/pci/pci.h"
#include "io/channel.h"
#include "hw/pci/memory-sync.h"
+#include "qemu/event_notifier.h"
#define TYPE_PCI_PROXY_DEV "pci-proxy-dev"
@@ -42,6 +43,10 @@ struct PCIProxyDev {
QIOChannel *ioc;
Error *migration_blocker;
RemoteMemSync sync;
+ int virq;
+ EventNotifier intr;
+ EventNotifier resample;
+
ProxyMemoryRegion region[PCI_NUM_REGIONS];
};
diff --git a/include/io/mpqemu-link.h b/include/io/mpqemu-link.h
index 234ac82d39..4cf81f5237 100644
--- a/include/io/mpqemu-link.h
+++ b/include/io/mpqemu-link.h
@@ -38,6 +38,7 @@ typedef enum {
PCI_CONFIG_READ,
BAR_WRITE,
BAR_READ,
+ SET_IRQFD,
MPQEMU_CMD_MAX,
} MPQemuCmd;
diff --git a/io/mpqemu-link.c b/io/mpqemu-link.c
index 44c2875a13..86f2e61ab8 100644
--- a/io/mpqemu-link.c
+++ b/io/mpqemu-link.c
@@ -283,6 +283,11 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)
return false;
}
break;
+ case SET_IRQFD:
+ if (msg->size || (msg->num_fds != 2)) {
+ return false;
+ }
+ break;
default:
break;
}
--
2.25.GIT
- [PATCH v10 03/19] multi-process: setup PCI host bridge for remote device, (continued)
- [PATCH v10 03/19] multi-process: setup PCI host bridge for remote device, elena . ufimtseva, 2020/10/05
- [PATCH v10 05/19] multi-process: add qio channel function to transmit, elena . ufimtseva, 2020/10/05
- [PATCH v10 08/19] multi-process: Associate fd of a PCIDevice with its object, elena . ufimtseva, 2020/10/05
- [PATCH v10 09/19] multi-process: setup memory manager for remote device, elena . ufimtseva, 2020/10/05
- [PATCH v10 12/19] multi-process: Forward PCI config space acceses to the remote process, elena . ufimtseva, 2020/10/05
- [PATCH v10 10/19] multi-process: introduce proxy object, elena . ufimtseva, 2020/10/05
- [PATCH v10 14/19] multi-process: Synchronize remote memory, elena . ufimtseva, 2020/10/05
- [PATCH v10 16/19] multi-process: Retrieve PCI info from remote process, elena . ufimtseva, 2020/10/05
- [PATCH v10 15/19] multi-process: create IOHUB object to handle irq,
elena . ufimtseva <=
- [PATCH v10 04/19] multi-process: setup a machine object for remote device process, elena . ufimtseva, 2020/10/05
- [PATCH v10 11/19] multi-process: add proxy communication functions, elena . ufimtseva, 2020/10/05
- [PATCH v10 13/19] multi-process: PCI BAR read/write handling for proxy & remote endpoints, elena . ufimtseva, 2020/10/05
- [PATCH v10 17/19] multi-process: perform device reset in the remote process, elena . ufimtseva, 2020/10/05
- [PATCH v10 19/19] multi-process: add configure and usage information, elena . ufimtseva, 2020/10/05
- [PATCH v10 18/19] multi-process: add the concept description to docs/devel/qemu-multiprocess, elena . ufimtseva, 2020/10/05
- Re: [PATCH v10 00/19] Initial support for multi-process Qemu, Stefan Hajnoczi, 2020/10/07