XHCI volatile and memory barriers

kernel/amd64/memorybarrier.h

@@ -0,0 +1,19 @@
+#ifndef _KERNEL_AMD64_MEMORYBARRIER_H
+#define _KERNEL_AMD64_MEMORYBARRIER_H
+
+#define memory_barrier()                                                                           \
+  do {                                                                                             \
+    __asm__ volatile ("mfence" ::: "memory");                                                      \
+  } while (0)
+
+#define read_memory_barrier()                                                                      \
+  do {                                                                                             \
+    __asm__ volatile ("lfence" ::: "memory");                                                      \
+  } while (0)
+
+#define write_memory_barrier()                                                                     \
+  do {                                                                                             \
+    __asm__ volatile ("sfence" ::: "memory");                                                      \
+  } while (0)
+
+#endif // _KERNEL_AMD64_MEMORYBARRIER_H

kernel/device/usb/xhci.c

@@ -15,6 +15,7 @@
 #include <proc/suspension_q.h>
 #include <status.h>
 #include <sys/debug.h>
+#include <sys/memorybarrier.h>
 #include <sys/spin_lock.h>
 #include <sys/stall.h>
 
@@ -510,6 +511,9 @@ static void xhci_endpoint0_put_trb (struct xhci_pdevice* pdevice, struct xhci_tr
     struct xhci_trb* link = &pdevice->endpoint0_ring.trbs[pdevice->endpoint0_ring.idx];
     link->param = pdevice->endpoint0_ring.phys;
     link->status = 0;
+
+    write_memory_barrier ();
+
     link->ctrl = (XHCI_TRB_LINK << XHCI_LNKTRB_CTRL_TRB_TYPE) |
         (1 << XHCI_LNKTRB_CTRL_TC) |
         pdevice->endpoint0_ring.cycle_bit;
@@ -521,6 +525,9 @@ static void xhci_endpoint0_put_trb (struct xhci_pdevice* pdevice, struct xhci_tr
   struct xhci_trb* trb = &pdevice->endpoint0_ring.trbs[pdevice->endpoint0_ring.idx];
   trb->param = put_trb.param;
   trb->status = put_trb.status;
+
+  write_memory_barrier ();
+
   trb->ctrl = (put_trb.ctrl & ~XHCI_LNKTRB_CTRL_TC) | pdevice->endpoint0_ring.cycle_bit;
 
   pdevice->endpoint0_ring.idx++;
@@ -544,6 +551,9 @@ static bool xhci_endpoint0_ctrl_in (struct xhci* xhci, struct xhci_pdevice* pdev
   memset (&trb, 0, sizeof (trb));
   trb.param = setup;
   trb.status = 8;
+
+  write_memory_barrier ();
+
   trb.ctrl = (XHCI_TRB_SETUP_STAGE << XHCI_SSTRB_CTRL_TRB_TYPE) |
       (1 << XHCI_SSTRB_CTRL_IDT) |
       (XHCI_SSTRB_TRT_IN_DATA_STAGE << XHCI_SSTRB_CTRL_TRT);
@@ -554,6 +564,9 @@ static bool xhci_endpoint0_ctrl_in (struct xhci* xhci, struct xhci_pdevice* pdev
   trb.param = data_phys;
   trb.status = length;
   trb.ctrl = (XHCI_TRB_DATA_STAGE << XHCI_DSTRB_CTRL_TRB_TYPE) | (1 << XHCI_DSTRB_CTRL_DIR);
+
+  write_memory_barrier ();
+
   xhci_endpoint0_put_trb (pdevice, trb);
 
   /* status stage */
@@ -561,6 +574,9 @@ static bool xhci_endpoint0_ctrl_in (struct xhci* xhci, struct xhci_pdevice* pdev
   trb.param = 0;
   trb.status = 0;
   trb.ctrl = (XHCI_TRB_STATUS_STAGE << XHCI_STSTRB_CTRL_TRB_TYPE) | (1 << XHCI_STSTRB_CTRL_IOC);
+
+  write_memory_barrier ();
+
   xhci_endpoint0_put_trb (pdevice, trb);
 
   atomic_store (&xhci->pending, true);
@@ -590,6 +606,9 @@ static void xhci_send_cmd (struct xhci* xhci, uint64_t param, uint32_t status, u
     struct xhci_trb* link = &xhci->cmd_ring.trbs[xhci->cmd_ring.idx];
     link->param = xhci->cmd_ring.phys;
     link->status = 0;
+
+    write_memory_barrier ();
+
     link->ctrl = (XHCI_TRB_LINK << XHCI_LNKTRB_CTRL_TRB_TYPE) |
         (1 << XHCI_LNKTRB_CTRL_TC) |
         xhci->cmd_ring.cycle_bit;
@@ -601,6 +620,9 @@ static void xhci_send_cmd (struct xhci* xhci, uint64_t param, uint32_t status, u
   struct xhci_trb* trb = &xhci->cmd_ring.trbs[xhci->cmd_ring.idx];
   trb->param = param;
   trb->status = status;
+
+  write_memory_barrier ();
+
   trb->ctrl = (ctrl & ~XHCI_LNKTRB_CTRL_TC) | xhci->cmd_ring.cycle_bit;
 
   xhci->cmd_ring.idx++;

kernel/device/usb/xhci.h

@@ -12,24 +12,24 @@
 
 /* event ring segment table entry */
 struct xhci_erst_entry {
-  uint64_t ptr;
-  uint32_t size;
-  uint32_t _rsvd;
+  volatile uint64_t ptr;
+  volatile uint32_t size;
+  volatile uint32_t _rsvd;
 } PACKED;
 
 /* transfer request block */
 struct xhci_trb {
-  uint64_t param;
-  uint32_t status;
-  uint32_t ctrl;
+  volatile uint64_t param;
+  volatile uint32_t status;
+  volatile uint32_t ctrl;
 } PACKED;
 
 struct xhci_ctx32 {
-  uint32_t dw[8];
+  volatile uint32_t dw[8];
 } PACKED;
 
 struct xhci_ctx64 {
-  uint32_t dw[16];
+  volatile uint32_t dw[16];
 } PACKED;
 
 struct xhci_input_ctx32 {

kernel/sys/memorybarrier.h

@@ -0,0 +1,8 @@
+#ifndef _KERNEL_SYS_MEMORYBARRIER_H
+#define _KERNEL_SYS_MEMORYBARRIER_H
+
+#if defined(__x86_64__)
+#include <amd64/memorybarrier.h>
+#endif
+
+#endif // _KERNEL_SYS_MEMORYBARRIER_H