usbredir: Add support for buffered bulk input (v2)

Buffered bulk mode is intended for bulk *input* endpoints, where the data is
of a streaming nature (not part of a command-response protocol). These
endpoints' input buffer may overflow if data is not read quickly enough.
So in buffered bulk mode the usb-host takes care of the submitting and
re-submitting of bulk transfers.

Buffered bulk mode is necessary for reliable operation with the bulk in
endpoints of usb to serial convertors. Unfortunatelty buffered bulk input
mode will only work with certain devices, therefor this patch also adds a
usb-id table to enable it for devices which need it, while leaving the
bulk ep handling for other devices unmodified.

Note that the bumping of the required usbredir from 0.5.3 to 0.6 does
not mean that we will now need a newer usbredir release then qemu-1.3,
.pc files reporting 0.5.3 have only ever existed in usbredir builds directly
from git, so qemu-1.3 needs the 0.6 release too.

Changes in v2:
-Split of quirk handling into quirks.c

Signed-off-by: Hans de Goede <hdegoede@redhat.com>

1 files changed, 337 insertions, 18 deletions

diff --git a/hw/usb/redirect.c b/hw/usb/redirect.c
index f15dc914ab..f1bf84c987 100644
--- a/hw/usb/redirect.c
+++ b/hw/usb/redirect.c
@@ -52,11 +52,13 @@
 
 typedef struct USBRedirDevice USBRedirDevice;
 
-/* Struct to hold buffered packets (iso or int input packets) */
+/* Struct to hold buffered packets */
 struct buf_packet {
     uint8_t *data;
-    int len;
-    int status;
+    void *free_on_destroy;
+    uint16_t len;
+    uint16_t offset;
+    uint8_t status;
     QTAILQ_ENTRY(buf_packet)next;
 };
 
@@ -70,11 +72,14 @@ struct endp_data {
     uint8_t iso_error; /* For reporting iso errors to the HC */
     uint8_t interrupt_started;
     uint8_t interrupt_error;
+    uint8_t bulk_receiving_enabled;
+    uint8_t bulk_receiving_started;
     uint8_t bufpq_prefilled;
     uint8_t bufpq_dropping_packets;
     QTAILQ_HEAD(, buf_packet) bufpq;
     int32_t bufpq_size;
     int32_t bufpq_target_size;
+    USBPacket *pending_async_packet;
 };
 
 struct PacketIdQueueEntry {
@@ -108,6 +113,7 @@ struct USBRedirDevice {
     struct endp_data endpoint[MAX_ENDPOINTS];
     struct PacketIdQueue cancelled;
     struct PacketIdQueue already_in_flight;
+    void (*buffered_bulk_in_complete)(USBRedirDevice *, USBPacket *, uint8_t);
     /* Data for device filtering */
     struct usb_redir_device_connect_header device_info;
     struct usb_redir_interface_info_header interface_info;
@@ -135,6 +141,8 @@ static void usbredir_interrupt_receiving_status(void *priv, uint64_t id,
     *interrupt_receiving_status);
 static void usbredir_bulk_streams_status(void *priv, uint64_t id,
     struct usb_redir_bulk_streams_status_header *bulk_streams_status);
+static void usbredir_bulk_receiving_status(void *priv, uint64_t id,
+    struct usb_redir_bulk_receiving_status_header *bulk_receiving_status);
 static void usbredir_control_packet(void *priv, uint64_t id,
     struct usb_redir_control_packet_header *control_packet,
     uint8_t *data, int data_len);
@@ -147,6 +155,9 @@ static void usbredir_iso_packet(void *priv, uint64_t id,
 static void usbredir_interrupt_packet(void *priv, uint64_t id,
     struct usb_redir_interrupt_packet_header *interrupt_header,
     uint8_t *data, int data_len);
+static void usbredir_buffered_bulk_packet(void *priv, uint64_t id,
+    struct usb_redir_buffered_bulk_packet_header *buffered_bulk_packet,
+    uint8_t *data, int data_len);
 
 static void usbredir_handle_status(USBRedirDevice *dev, USBPacket *p,
     int status);
@@ -320,12 +331,19 @@ static void packet_id_queue_empty(struct PacketIdQueue *q)
 static void usbredir_cancel_packet(USBDevice *udev, USBPacket *p)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    int i = USBEP2I(p->ep);
 
     if (p->combined) {
         usb_combined_packet_cancel(udev, p);
         return;
     }
 
+    if (dev->endpoint[i].pending_async_packet) {
+        assert(dev->endpoint[i].pending_async_packet == p);
+        dev->endpoint[i].pending_async_packet = NULL;
+        return;
+    }
+
     packet_id_queue_add(&dev->cancelled, p->id);
     usbredirparser_send_cancel_data_packet(dev->parser, p->id);
     usbredirparser_do_write(dev->parser);
@@ -344,6 +362,11 @@ static void usbredir_fill_already_in_flight_from_ep(USBRedirDevice *dev,
 {
     static USBPacket *p;
 
+    /* async handled packets for bulk receiving eps do not count as inflight */
+    if (dev->endpoint[USBEP2I(ep)].bulk_receiving_started) {
+        return;
+    }
+
     QTAILQ_FOREACH(p, &ep->queue, queue) {
         /* Skip combined packets, except for the first */
         if (p->combined && p != p->combined->first) {
@@ -391,8 +414,8 @@ static USBPacket *usbredir_find_packet_by_id(USBRedirDevice *dev,
     return p;
 }
 
-static void bufp_alloc(USBRedirDevice *dev,
-    uint8_t *data, int len, int status, uint8_t ep)
+static void bufp_alloc(USBRedirDevice *dev, uint8_t *data, uint16_t len,
+    uint8_t status, uint8_t ep, void *free_on_destroy)
 {
     struct buf_packet *bufp;
 
@@ -416,7 +439,9 @@ static void bufp_alloc(USBRedirDevice *dev,
     bufp = g_malloc(sizeof(struct buf_packet));
     bufp->data   = data;
     bufp->len    = len;
+    bufp->offset = 0;
     bufp->status = status;
+    bufp->free_on_destroy = free_on_destroy;
     QTAILQ_INSERT_TAIL(&dev->endpoint[EP2I(ep)].bufpq, bufp, next);
     dev->endpoint[EP2I(ep)].bufpq_size++;
 }
@@ -426,7 +451,7 @@ static void bufp_free(USBRedirDevice *dev, struct buf_packet *bufp,
 {
     QTAILQ_REMOVE(&dev->endpoint[EP2I(ep)].bufpq, bufp, next);
     dev->endpoint[EP2I(ep)].bufpq_size--;
-    free(bufp->data);
+    free(bufp->free_on_destroy);
     g_free(bufp);
 }
 
@@ -577,19 +602,162 @@ static void usbredir_stop_iso_stream(USBRedirDevice *dev, uint8_t ep)
     usbredir_free_bufpq(dev, ep);
 }
 
+/*
+ * The usb-host may poll the endpoint faster then our guest, resulting in lots
+ * of smaller bulkp-s. The below buffered_bulk_in_complete* functions combine
+ * data from multiple bulkp-s into a single packet, avoiding bufpq overflows.
+ */
+static void usbredir_buffered_bulk_add_data_to_packet(USBRedirDevice *dev,
+    struct buf_packet *bulkp, int count, USBPacket *p, uint8_t ep)
+{
+    usb_packet_copy(p, bulkp->data + bulkp->offset, count);
+    bulkp->offset += count;
+    if (bulkp->offset == bulkp->len) {
+        /* Store status in the last packet with data from this bulkp */
+        usbredir_handle_status(dev, p, bulkp->status);
+        bufp_free(dev, bulkp, ep);
+    }
+}
+
+static void usbredir_buffered_bulk_in_complete_raw(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    struct buf_packet *bulkp;
+    int count;
+
+    while ((bulkp = QTAILQ_FIRST(&dev->endpoint[EP2I(ep)].bufpq)) &&
+           p->actual_length < p->iov.size && p->status == USB_RET_SUCCESS) {
+        count = bulkp->len - bulkp->offset;
+        if (count > (p->iov.size - p->actual_length)) {
+            count = p->iov.size - p->actual_length;
+        }
+        usbredir_buffered_bulk_add_data_to_packet(dev, bulkp, count, p, ep);
+    }
+}
+
+static void usbredir_buffered_bulk_in_complete_ftdi(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    const int maxp = dev->endpoint[EP2I(ep)].max_packet_size;
+    uint8_t header[2] = { 0, 0 };
+    struct buf_packet *bulkp;
+    int count;
+
+    while ((bulkp = QTAILQ_FIRST(&dev->endpoint[EP2I(ep)].bufpq)) &&
+           p->actual_length < p->iov.size && p->status == USB_RET_SUCCESS) {
+        if (bulkp->len < 2) {
+            WARNING("malformed ftdi bulk in packet\n");
+            bufp_free(dev, bulkp, ep);
+            continue;
+        }
+
+        if ((p->actual_length % maxp) == 0) {
+            usb_packet_copy(p, bulkp->data, 2);
+            memcpy(header, bulkp->data, 2);
+        } else {
+            if (bulkp->data[0] != header[0] || bulkp->data[1] != header[1]) {
+                break; /* Different header, add to next packet */
+            }
+        }
+
+        if (bulkp->offset == 0) {
+            bulkp->offset = 2; /* Skip header */
+        }
+        count = bulkp->len - bulkp->offset;
+        /* Must repeat the header at maxp interval */
+        if (count > (maxp - (p->actual_length % maxp))) {
+            count = maxp - (p->actual_length % maxp);
+        }
+        usbredir_buffered_bulk_add_data_to_packet(dev, bulkp, count, p, ep);
+    }
+}
+
+static void usbredir_buffered_bulk_in_complete(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
+    dev->buffered_bulk_in_complete(dev, p, ep);
+    DPRINTF("bulk-token-in ep %02X status %d len %d id %"PRIu64"\n",
+            ep, p->status, p->actual_length, p->id);
+}
+
+static void usbredir_handle_buffered_bulk_in_data(USBRedirDevice *dev,
+    USBPacket *p, uint8_t ep)
+{
+    /* Input bulk endpoint, buffered packet input */
+    if (!dev->endpoint[EP2I(ep)].bulk_receiving_started) {
+        int bpt;
+        struct usb_redir_start_bulk_receiving_header start = {
+            .endpoint = ep,
+            .stream_id = 0,
+            .no_transfers = 5,
+        };
+        /* Round bytes_per_transfer up to a multiple of max_packet_size */
+        bpt = 512 + dev->endpoint[EP2I(ep)].max_packet_size - 1;
+        bpt /= dev->endpoint[EP2I(ep)].max_packet_size;
+        bpt *= dev->endpoint[EP2I(ep)].max_packet_size;
+        start.bytes_per_transfer = bpt;
+        /* No id, we look at the ep when receiving a status back */
+        usbredirparser_send_start_bulk_receiving(dev->parser, 0, &start);
+        usbredirparser_do_write(dev->parser);
+        DPRINTF("bulk receiving started bytes/transfer %u count %d ep %02X\n",
+                start.bytes_per_transfer, start.no_transfers, ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_started = 1;
+        /* We don't really want to drop bulk packets ever, but
+           having some upper limit to how much we buffer is good. */
+        dev->endpoint[EP2I(ep)].bufpq_target_size = 5000;
+        dev->endpoint[EP2I(ep)].bufpq_dropping_packets = 0;
+    }
+
+    if (QTAILQ_EMPTY(&dev->endpoint[EP2I(ep)].bufpq)) {
+        DPRINTF("bulk-token-in ep %02X, no bulkp\n", ep);
+        assert(dev->endpoint[EP2I(ep)].pending_async_packet == NULL);
+        dev->endpoint[EP2I(ep)].pending_async_packet = p;
+        p->status = USB_RET_ASYNC;
+        return;
+    }
+    usbredir_buffered_bulk_in_complete(dev, p, ep);
+}
+
+static void usbredir_stop_bulk_receiving(USBRedirDevice *dev, uint8_t ep)
+{
+    struct usb_redir_stop_bulk_receiving_header stop_bulk = {
+        .endpoint = ep,
+        .stream_id = 0,
+    };
+    if (dev->endpoint[EP2I(ep)].bulk_receiving_started) {
+        usbredirparser_send_stop_bulk_receiving(dev->parser, 0, &stop_bulk);
+        DPRINTF("bulk receiving stopped ep %02X\n", ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_started = 0;
+    }
+    usbredir_free_bufpq(dev, ep);
+}
+
 static void usbredir_handle_bulk_data(USBRedirDevice *dev, USBPacket *p,
                                       uint8_t ep)
 {
     struct usb_redir_bulk_packet_header bulk_packet;
     size_t size = (p->combined) ? p->combined->iov.size : p->iov.size;
-
-    DPRINTF("bulk-out ep %02X len %zd id %"PRIu64"\n", ep, size, p->id);
+    const int maxp = dev->endpoint[EP2I(ep)].max_packet_size;
 
     if (usbredir_already_in_flight(dev, p->id)) {
         p->status = USB_RET_ASYNC;
         return;
     }
 
+    if (dev->endpoint[EP2I(ep)].bulk_receiving_enabled) {
+        if (size != 0 && (size % maxp) == 0) {
+            usbredir_handle_buffered_bulk_in_data(dev, p, ep);
+            return;
+        }
+        WARNING("bulk recv invalid size %zd ep %02x, disabling\n", size, ep);
+        assert(dev->endpoint[EP2I(ep)].pending_async_packet == NULL);
+        usbredir_stop_bulk_receiving(dev, ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_enabled = 0;
+    }
+
+    DPRINTF("bulk-out ep %02X len %zd id %"PRIu64"\n", ep, size, p->id);
+
     bulk_packet.endpoint  = ep;
     bulk_packet.length    = size;
     bulk_packet.stream_id = 0;
@@ -726,9 +894,6 @@ static void usbredir_handle_data(USBDevice *udev, USBPacket *p)
         ERROR("handle_data called for control transfer on ep %02X\n", ep);
         p->status = USB_RET_NAK;
         break;
-    case USB_ENDPOINT_XFER_ISOC:
-        usbredir_handle_iso_data(dev, p, ep);
-        break;
     case USB_ENDPOINT_XFER_BULK:
         if (p->state == USB_PACKET_SETUP && p->pid == USB_TOKEN_IN &&
                 p->ep->pipeline) {
@@ -737,6 +902,9 @@ static void usbredir_handle_data(USBDevice *udev, USBPacket *p)
         }
         usbredir_handle_bulk_data(dev, p, ep);
         break;
+    case USB_ENDPOINT_XFER_ISOC:
+        usbredir_handle_iso_data(dev, p, ep);
+        break;
     case USB_ENDPOINT_XFER_INT:
         if (ep & USB_DIR_IN) {
             usbredir_handle_interrupt_in_data(dev, p, ep);
@@ -763,6 +931,11 @@ static void usbredir_stop_ep(USBRedirDevice *dev, int i)
     uint8_t ep = I2EP(i);
 
     switch (dev->endpoint[i].type) {
+    case USB_ENDPOINT_XFER_BULK:
+        if (ep & USB_DIR_IN) {
+            usbredir_stop_bulk_receiving(dev, ep);
+        }
+        break;
     case USB_ENDPOINT_XFER_ISOC:
         usbredir_stop_iso_stream(dev, ep);
         break;
@@ -942,10 +1115,12 @@ static void usbredir_create_parser(USBRedirDevice *dev)
     dev->parser->interrupt_receiving_status_func =
         usbredir_interrupt_receiving_status;
     dev->parser->bulk_streams_status_func = usbredir_bulk_streams_status;
+    dev->parser->bulk_receiving_status_func = usbredir_bulk_receiving_status;
     dev->parser->control_packet_func = usbredir_control_packet;
     dev->parser->bulk_packet_func = usbredir_bulk_packet;
     dev->parser->iso_packet_func = usbredir_iso_packet;
     dev->parser->interrupt_packet_func = usbredir_interrupt_packet;
+    dev->parser->buffered_bulk_packet_func = usbredir_buffered_bulk_packet;
     dev->read_buf = NULL;
     dev->read_buf_size = 0;
 
@@ -954,6 +1129,7 @@ static void usbredir_create_parser(USBRedirDevice *dev)
     usbredirparser_caps_set_cap(caps, usb_redir_cap_ep_info_max_packet_size);
     usbredirparser_caps_set_cap(caps, usb_redir_cap_64bits_ids);
     usbredirparser_caps_set_cap(caps, usb_redir_cap_32bits_bulk_length);
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_bulk_receiving);
 
     if (runstate_check(RUN_STATE_INMIGRATE)) {
         flags |= usbredirparser_fl_no_hello;
@@ -1192,6 +1368,52 @@ error:
     return -1;
 }
 
+static void usbredir_check_bulk_receiving(USBRedirDevice *dev)
+{
+    int i, j, quirks;
+
+    if (!usbredirparser_peer_has_cap(dev->parser,
+                                     usb_redir_cap_bulk_receiving)) {
+        return;
+    }
+
+    for (i = EP2I(USB_DIR_IN); i < MAX_ENDPOINTS; i++) {
+        dev->endpoint[i].bulk_receiving_enabled = 0;
+    }
+    for (i = 0; i < dev->interface_info.interface_count; i++) {
+        quirks = usb_get_quirks(dev->device_info.vendor_id,
+                                dev->device_info.product_id,
+                                dev->interface_info.interface_class[i],
+                                dev->interface_info.interface_subclass[i],
+                                dev->interface_info.interface_protocol[i]);
+        if (!(quirks & USB_QUIRK_BUFFER_BULK_IN)) {
+            continue;
+        }
+        if (quirks & USB_QUIRK_IS_FTDI) {
+            dev->buffered_bulk_in_complete =
+                usbredir_buffered_bulk_in_complete_ftdi;
+        } else {
+            dev->buffered_bulk_in_complete =
+                usbredir_buffered_bulk_in_complete_raw;
+        }
+
+        for (j = EP2I(USB_DIR_IN); j < MAX_ENDPOINTS; j++) {
+            if (dev->endpoint[j].interface ==
+                                    dev->interface_info.interface[i] &&
+                    dev->endpoint[j].type == USB_ENDPOINT_XFER_BULK &&
+                    dev->endpoint[j].max_packet_size != 0) {
+                dev->endpoint[j].bulk_receiving_enabled = 1;
+                /*
+                 * With buffering pipelining is not necessary. Also packet
+                 * combining and bulk in buffering don't play nice together!
+                 */
+                I2USBEP(dev, j)->pipeline = false;
+                break; /* Only buffer for the first ep of each intf */
+            }
+        }
+    }
+}
+
 /*
  * usbredirparser packet complete callbacks
  */
@@ -1300,6 +1522,7 @@ static void usbredir_device_connect(void *priv,
         return;
     }
 
+    usbredir_check_bulk_receiving(dev);
     qemu_mod_timer(dev->attach_timer, dev->next_attach_time);
 }
 
@@ -1338,9 +1561,10 @@ static void usbredir_interface_info(void *priv,
 
     /*
      * If we receive interface info after the device has already been
-     * connected (ie on a set_config), re-check the filter.
+     * connected (ie on a set_config), re-check interface dependent things.
      */
     if (qemu_timer_pending(dev->attach_timer) || dev->dev.attached) {
+        usbredir_check_bulk_receiving(dev);
         if (usbredir_check_filter(dev)) {
             ERROR("Device no longer matches filter after interface info "
                   "change, disconnecting!\n");
@@ -1441,6 +1665,7 @@ static void usbredir_ep_info(void *priv,
         return;
     }
     usbredir_setup_usb_eps(dev);
+    usbredir_check_bulk_receiving(dev);
 }
 
 static void usbredir_configuration_status(void *priv, uint64_t id,
@@ -1531,6 +1756,25 @@ static void usbredir_bulk_streams_status(void *priv, uint64_t id,
 {
 }
 
+static void usbredir_bulk_receiving_status(void *priv, uint64_t id,
+    struct usb_redir_bulk_receiving_status_header *bulk_receiving_status)
+{
+    USBRedirDevice *dev = priv;
+    uint8_t ep = bulk_receiving_status->endpoint;
+
+    DPRINTF("bulk recv status %d ep %02X id %"PRIu64"\n",
+            bulk_receiving_status->status, ep, id);
+
+    if (!dev->dev.attached || !dev->endpoint[EP2I(ep)].bulk_receiving_started) {
+        return;
+    }
+
+    if (bulk_receiving_status->status == usb_redir_stall) {
+        DPRINTF("bulk receiving stopped by peer ep %02X\n", ep);
+        dev->endpoint[EP2I(ep)].bulk_receiving_started = 0;
+    }
+}
+
 static void usbredir_control_packet(void *priv, uint64_t id,
     struct usb_redir_control_packet_header *control_packet,
     uint8_t *data, int data_len)
@@ -1636,7 +1880,7 @@ static void usbredir_iso_packet(void *priv, uint64_t id,
     }
 
     /* bufp_alloc also adds the packet to the ep queue */
-    bufp_alloc(dev, data, data_len, iso_packet->status, ep);
+    bufp_alloc(dev, data, data_len, iso_packet->status, ep, data);
 }
 
 static void usbredir_interrupt_packet(void *priv, uint64_t id,
@@ -1667,7 +1911,7 @@ static void usbredir_interrupt_packet(void *priv, uint64_t id,
         }
 
         /* bufp_alloc also adds the packet to the ep queue */
-        bufp_alloc(dev, data, data_len, interrupt_packet->status, ep);
+        bufp_alloc(dev, data, data_len, interrupt_packet->status, ep, data);
     } else {
         /*
          * We report output interrupt packets as completed directly upon
@@ -1680,6 +1924,52 @@ static void usbredir_interrupt_packet(void *priv, uint64_t id,
     }
 }
 
+static void usbredir_buffered_bulk_packet(void *priv, uint64_t id,
+    struct usb_redir_buffered_bulk_packet_header *buffered_bulk_packet,
+    uint8_t *data, int data_len)
+{
+    USBRedirDevice *dev = priv;
+    uint8_t status, ep = buffered_bulk_packet->endpoint;
+    void *free_on_destroy;
+    int i, len;
+
+    DPRINTF("buffered-bulk-in status %d ep %02X len %d id %"PRIu64"\n",
+            buffered_bulk_packet->status, ep, data_len, id);
+
+    if (dev->endpoint[EP2I(ep)].type != USB_ENDPOINT_XFER_BULK) {
+        ERROR("received buffered-bulk packet for non bulk ep %02X\n", ep);
+        free(data);
+        return;
+    }
+
+    if (dev->endpoint[EP2I(ep)].bulk_receiving_started == 0) {
+        DPRINTF("received buffered-bulk packet on not started ep %02X\n", ep);
+        free(data);
+        return;
+    }
+
+    /* Data must be in maxp chunks for buffered_bulk_add_*_data_to_packet */
+    len = dev->endpoint[EP2I(ep)].max_packet_size;
+    status = usb_redir_success;
+    free_on_destroy = NULL;
+    for (i = 0; i < data_len; i += len) {
+        if (len >= (data_len - i)) {
+            len = data_len - i;
+            status = buffered_bulk_packet->status;
+            free_on_destroy = data;
+        }
+        /* bufp_alloc also adds the packet to the ep queue */
+        bufp_alloc(dev, data + i, len, status, ep, free_on_destroy);
+    }
+
+    if (dev->endpoint[EP2I(ep)].pending_async_packet) {
+        USBPacket *p = dev->endpoint[EP2I(ep)].pending_async_packet;
+        dev->endpoint[EP2I(ep)].pending_async_packet = NULL;
+        usbredir_buffered_bulk_in_complete(dev, p, ep);
+        usb_packet_complete(&dev->dev, p);
+    }
+}
+
 /*
  * Migration code
  */
@@ -1714,6 +2004,7 @@ static int usbredir_post_load(void *priv, int version_id)
     dev->dev.speedmask = (1 << dev->dev.speed);
 
     usbredir_setup_usb_eps(dev);
+    usbredir_check_bulk_receiving(dev);
 
     return 0;
 }
@@ -1787,15 +2078,16 @@ static void usbredir_put_bufpq(QEMUFile *f, void *priv, size_t unused)
     struct endp_data *endp = priv;
     USBRedirDevice *dev = endp->dev;
     struct buf_packet *bufp;
-    int i = 0;
+    int len, i = 0;
 
     qemu_put_be32(f, endp->bufpq_size);
     QTAILQ_FOREACH(bufp, &endp->bufpq, next) {
+        len = bufp->len - bufp->offset;
         DPRINTF("put_bufpq %d/%d len %d status %d\n", i + 1, endp->bufpq_size,
-                bufp->len, bufp->status);
-        qemu_put_be32(f, bufp->len);
+                len, bufp->status);
+        qemu_put_be32(f, len);
         qemu_put_be32(f, bufp->status);
-        qemu_put_buffer(f, bufp->data, bufp->len);
+        qemu_put_buffer(f, bufp->data + bufp->offset, len);
         i++;
     }
     assert(i == endp->bufpq_size);
@@ -1813,7 +2105,9 @@ static int usbredir_get_bufpq(QEMUFile *f, void *priv, size_t unused)
         bufp = g_malloc(sizeof(struct buf_packet));
         bufp->len = qemu_get_be32(f);
         bufp->status = qemu_get_be32(f);
+        bufp->offset = 0;
         bufp->data = qemu_oom_check(malloc(bufp->len)); /* regular malloc! */
+        bufp->free_on_destroy = bufp->data;
         qemu_get_buffer(f, bufp->data, bufp->len);
         QTAILQ_INSERT_TAIL(&endp->bufpq, bufp, next);
         DPRINTF("get_bufpq %d/%d len %d status %d\n", i + 1, endp->bufpq_size,
@@ -1830,6 +2124,23 @@ static const VMStateInfo usbredir_ep_bufpq_vmstate_info = {
 
 
 /* For endp_data migration */
+static const VMStateDescription usbredir_bulk_receiving_vmstate = {
+    .name = "usb-redir-ep/bulk-receiving",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(bulk_receiving_started, struct endp_data),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool usbredir_bulk_receiving_needed(void *priv)
+{
+    struct endp_data *endp = priv;
+
+    return endp->bulk_receiving_started;
+}
+
 static const VMStateDescription usbredir_ep_vmstate = {
     .name = "usb-redir-ep",
     .version_id = 1,
@@ -1856,6 +2167,14 @@ static const VMStateDescription usbredir_ep_vmstate = {
         },
         VMSTATE_INT32(bufpq_target_size, struct endp_data),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &usbredir_bulk_receiving_vmstate,
+            .needed = usbredir_bulk_receiving_needed,
+        }, {
+            /* empty */
+        }
     }
 };