From c048be5cc92ae201c339d46984476c4629275ed6 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Tue, 5 Aug 2014 13:05:52 -0600 Subject: vfio: Fix MSI-X vector expansion When new MSI-X vectors are enabled we need to disable MSI-X and re-enable it with the correct number of vectors. That means we need to reprogram the eventfd triggers for each vector. Prior to f4d45d47 vector->use tracked whether a vector was masked or unmasked and we could always pick the KVM path when available for unmasked vectors. Now vfio doesn't track mask state itself and vector->use and virq remains configured even for masked vectors. Therefore we need to ask the MSI-X code whether a vector is masked in order to select the correct signaling path. As noted in the comment, MSI relies on hardware to handle masking. Signed-off-by: Alex Williamson Cc: qemu-stable@nongnu.org # QEMU 2.1 --- hw/misc/vfio.c | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/hw/misc/vfio.c b/hw/misc/vfio.c index 0b9eba0c84..e88b610721 100644 --- a/hw/misc/vfio.c +++ b/hw/misc/vfio.c @@ -120,11 +120,20 @@ typedef struct VFIOINTx { } VFIOINTx; typedef struct VFIOMSIVector { - EventNotifier interrupt; /* eventfd triggered on interrupt */ - EventNotifier kvm_interrupt; /* eventfd triggered for KVM irqfd bypass */ + /* + * Two interrupt paths are configured per vector. The first, is only used + * for interrupts injected via QEMU. This is typically the non-accel path, + * but may also be used when we want QEMU to handle masking and pending + * bits. The KVM path bypasses QEMU and is therefore higher performance, + * but requires masking at the device. virq is used to track the MSI route + * through KVM, thus kvm_interrupt is only available when virq is set to a + * valid (>= 0) value. + */ + EventNotifier interrupt; + EventNotifier kvm_interrupt; struct VFIODevice *vdev; /* back pointer to device */ MSIMessage msg; /* cache the MSI message so we know when it changes */ - int virq; /* KVM irqchip route for QEMU bypass */ + int virq; bool use; } VFIOMSIVector; @@ -681,13 +690,24 @@ static int vfio_enable_vectors(VFIODevice *vdev, bool msix) fds = (int32_t *)&irq_set->data; for (i = 0; i < vdev->nr_vectors; i++) { - if (!vdev->msi_vectors[i].use) { - fds[i] = -1; - } else if (vdev->msi_vectors[i].virq >= 0) { - fds[i] = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt); - } else { - fds[i] = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt); + int fd = -1; + + /* + * MSI vs MSI-X - The guest has direct access to MSI mask and pending + * bits, therefore we always use the KVM signaling path when setup. + * MSI-X mask and pending bits are emulated, so we want to use the + * KVM signaling path only when configured and unmasked. + */ + if (vdev->msi_vectors[i].use) { + if (vdev->msi_vectors[i].virq < 0 || + (msix && msix_is_masked(&vdev->pdev, i))) { + fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt); + } else { + fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt); + } } + + fds[i] = fd; } ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set); -- cgit v1.2.1 From 9b3af4c0e40f14b5173ad6e2d9fcc376207ec1dd Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Tue, 5 Aug 2014 13:05:57 -0600 Subject: vfio: Don't cache MSIMessage Commit 40509f7f added a test to avoid updating KVM MSI routes when the MSIMessage is unchanged and f4d45d47 switched to relying on this rather than doing our own comparison. Our cached msg is effectively unused now. Remove it. Signed-off-by: Alex Williamson --- hw/misc/vfio.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/hw/misc/vfio.c b/hw/misc/vfio.c index e88b610721..ba08adbd99 100644 --- a/hw/misc/vfio.c +++ b/hw/misc/vfio.c @@ -132,7 +132,6 @@ typedef struct VFIOMSIVector { EventNotifier interrupt; EventNotifier kvm_interrupt; struct VFIODevice *vdev; /* back pointer to device */ - MSIMessage msg; /* cache the MSI message so we know when it changes */ int virq; bool use; } VFIOMSIVector; @@ -744,7 +743,6 @@ static void vfio_add_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage *msg, return; } - vector->msg = *msg; vector->virq = virq; } @@ -760,7 +758,6 @@ static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector) static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg) { kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg); - vector->msg = msg; } static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr, @@ -939,6 +936,7 @@ retry: for (i = 0; i < vdev->nr_vectors; i++) { VFIOMSIVector *vector = &vdev->msi_vectors[i]; + MSIMessage msg = msi_get_message(&vdev->pdev, i); vector->vdev = vdev; vector->virq = -1; @@ -951,13 +949,11 @@ retry: qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt), vfio_msi_interrupt, NULL, vector); - vector->msg = msi_get_message(&vdev->pdev, i); - /* * Attempt to enable route through KVM irqchip, * default to userspace handling if unavailable. */ - vfio_add_kvm_msi_virq(vector, &vector->msg, false); + vfio_add_kvm_msi_virq(vector, &msg, false); } /* Set interrupt type prior to possible interrupts */ -- cgit v1.2.1