hw/usb/canokey: fix compatibility of qemu-xhci

XHCI wont poll interrupt IN endpoint if NAKed, and needs wakeup

Suggested-by: Gerd Hoffmann <kraxel@redhat.com>
Signed-off-by: Hongren (Zenithal) Zheng <i@zenithal.me>
Message-Id: <YqcqSHNpI7sXRNpZ@Sun>
Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>

1 files changed, 23 insertions, 0 deletions

diff --git a/hw/usb/canokey.c b/hw/usb/canokey.c
index 8654892..8da0d65 100644
--- a/hw/usb/canokey.c
+++ b/hw/usb/canokey.c
@@ -104,6 +104,13 @@ int canokey_emu_transmit(
     key->ep_in_size[ep_in] += size;
     key->ep_in_state[ep_in] = CANOKEY_EP_IN_READY;
     /*
+     * wake up controller if we NAKed IN token before
+     * Note: this is a quirk for CanoKey CTAPHID
+     */
+    if (ep_in == CANOKEY_EMU_EP_CTAPHID) {
+        usb_wakeup(usb_ep_get(&key->dev, USB_TOKEN_IN, ep_in), 0);
+    }
+    /*
      * ready for more data in device loop
      *
      * Note: this is a quirk for CanoKey CTAPHID
@@ -208,6 +215,22 @@ static void canokey_handle_data(USBDevice *dev, USBPacket *p)
             key->ep_out_size[ep_out] = out_len;
             canokey_emu_data_out(ep_out, NULL);
         }
+        /*
+         * Note: this is a quirk for CanoKey CTAPHID
+         *
+         * There is one code path that uses this device loop
+         * INTR IN -> useful data_in and useless device_loop -> NAKed
+         * INTR OUT -> useful device loop -> transmit -> wakeup
+         *   (useful thanks to both data_in and data_out having been called)
+         * the next INTR IN -> actual data to guest
+         *
+         * if there is no such device loop, there would be no further
+         * INTR IN, no device loop, no transmit hence no usb_wakeup
+         * then qemu would hang
+         */
+        if (ep_in == CANOKEY_EMU_EP_CTAPHID) {
+            canokey_emu_device_loop(); /* may call transmit multiple times */
+        }
         break;
     case USB_TOKEN_IN:
         if (key->ep_in_pos[ep_in] == 0) { /* first time IN */