#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* REF: * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf */ /* clang-format off */ /* capability registers */ #define XHCI_CAPLENGTH 0x00 #define XHCI_RSVD 0x01 #define XHCI_HCIVERSION 0x02 #define XHCI_HCSPARAMS1 0x04 #define XHCI_HCSPARAMS2 0x08 #define XHCI_HCSPARAMS3 0x0C #define XHCI_HCCPARAMS1 0x10 #define XHCI_DBOFF 0x14 #define XHCI_RTSOFF 0x18 #define XHCI_HCCPARAMS2 0x1C /* operational registers */ #define XHCI_USBCMD 0x00 #define XHCI_USBSTS 0x04 #define XHCI_PAGESIZE 0x08 #define XHCI_DNCTRL 0x14 #define XHCI_CRCR 0x18 #define XHCI_DCBAAP 0x30 #define XHCI_CONFIG 0x38 /* port registers */ #define XHCI_PORTSC 0x00 #define XHCI_PORTPMSC 0x04 #define XHCI_PORTLI 0x08 /* runtime registers */ #define XHCI_MFINDEX 0x00 /* + IRQ sets (0x20) */ #define XHCI_IMAN 0x00 #define XHCI_IMOD 0x04 #define XHCI_ERSTSZ 0x08 #define XHCI_ERSTBA 0x10 #define XHCI_ERDP 0x18 /* event types */ #define XHCI_TRB_NORMAL 1 #define XHCI_TRB_SETUP_STAGE 2 #define XHCI_TRB_DATA_STAGE 3 #define XHCI_TRB_STATUS_STAGE 4 #define XHCI_TRB_ISOCH 5 #define XHCI_TRB_LINK 6 #define XHCI_TRB_EVENT_DATA 7 #define XHCI_TRB_NOOP 8 #define XHCI_TRB_SLOT_ENAB_CMD 9 #define XHCI_TRB_SLOT_DISB_CMD 10 #define XHCI_TRB_ADDR_DEV_CMD 11 #define XHCI_TRB_CFG_ENDP_CMD 12 #define XHCI_TRB_EVAL_CTX_CMD 13 #define XHCI_TRB_RESET_ENDP_CMD 14 #define XHCI_TRB_STOP_ENDP_CMD 15 #define XHCI_TRB_SET_DRDQP_CMD 16 #define XHCI_TRB_RESET_DEV_CMD 17 #define XHCI_TRB_FORCE_EVT_CMD 18 #define XHCI_TRB_NEGO_BANDW_CMD 19 #define XHCI_TRB_SET_LTV_CMD 20 #define XHCI_TRB_PORT_BANDW_CMD 21 #define XHCI_TRB_FORCE_HEADER 22 #define XHCI_TRB_NOOP_CMD 23 #define XHCI_TRB_GET_EXTPRP_CMD 24 #define XHCI_TRB_SET_EXTPRP_CMD 25 #define XHCI_TRB_TRANSFER_EVENT 32 #define XHCI_TRB_CMD_CMPL_EVENT 33 #define XHCI_TRB_PORT_STS_CHNG 34 #define XHCI_TRB_BANDW_RQ_EVENT 35 #define XHCI_TRB_DOORBELL_EVENT 36 #define XHCI_TRB_HOST_CTRL_EVNT 37 #define XHCI_TRB_DEV_NOTIF_EVNT 38 #define XHCI_TRB_MFINDEX_WRAP 39 /* clang-format on */ static void xhci_write8 (uintptr_t base, uint32_t reg, uint8_t value) { *(volatile uint8_t*)(base + reg) = value; } static void xhci_write16 (uintptr_t base, uint32_t reg, uint16_t value) { *(volatile uint16_t*)(base + reg) = value; } static void xhci_write32 (uintptr_t base, uint32_t reg, uint32_t value) { *(volatile uint32_t*)(base + reg) = value; } static uint8_t xhci_read8 (uintptr_t base, uint32_t reg) { return *(volatile uint8_t*)(base + reg); } static uint16_t xhci_read16 (uintptr_t base, uint32_t reg) { return *(volatile uint16_t*)(base + reg); } static uint32_t xhci_read32 (uintptr_t base, uint32_t reg) { return *(volatile uint32_t*)(base + reg); } static void xhci_event_dispatch (struct xhci* xhci, struct xhci_trb* event, uint8_t type) { switch (type) { case XHCI_TRB_CMD_CMPL_EVENT: { uintptr_t cmd_trb_phys = event->param; uint8_t cmpl_code = (event->status >> 24) & 0xFF; uint8_t slot_id = (event->ctrl >> 24) & 0xFF; DEBUG ("cmd completion: phys=%p,code=%u,slot=%u\n", cmd_trb_phys, cmpl_code, slot_id); } break; default: DEBUG ("Unhandled event type %u at %u\n", type, xhci->event_ring_idx); break; } } static void xhci_poll_events (struct xhci* xhci) { uintptr_t ir_base = xhci->xhci_runtime_base + 0x20; bool serviced = false; for (;;) { struct xhci_trb* event = &xhci->event_ring[xhci->event_ring_idx]; if ((event->ctrl & 0x1) != xhci->event_cycle_bit) { break; } serviced = true; uint8_t type = (event->ctrl >> 10) & 0x3F; xhci_event_dispatch (xhci, event, type); xhci->event_ring_idx++; if (xhci->event_ring_idx >= xhci->event_ring_size) { xhci->event_ring_idx = 0; xhci->event_cycle_bit ^= 1; } } if (serviced) { uintptr_t dequeue_ptr = xhci->event_ring_phys + (xhci->event_ring_idx * sizeof (struct xhci_trb)); xhci_write32 (ir_base, XHCI_ERDP, (uint32_t)dequeue_ptr | (1 << 3)); xhci_write32 (ir_base, XHCI_ERDP + 4, (uint32_t)(dequeue_ptr >> 32)); atomic_store (&xhci->pending, false); } } static void xhci_irq (void* arg, void* regs, bool user, struct reschedule_ctx* rctx) { (void)user, (void)regs, (void)rctx; uint64_t fd; struct xhci* xhci = arg; spin_lock (&xhci->device->lock, &fd); uintptr_t ir_base = xhci->xhci_runtime_base + 0x20; /* ack */ uint32_t iman = xhci_read32 (ir_base, XHCI_IMAN); xhci_write32 (ir_base, XHCI_IMAN, iman | 0x01); uint32_t usbsts = xhci_read32 (xhci->xhci_oper_base, XHCI_USBSTS); xhci_write32 (xhci->xhci_oper_base, XHCI_USBSTS, usbsts | (1 << 3)); /* clear event busy */ uint64_t erdp = (uint64_t)xhci_read32 (ir_base, XHCI_ERDP) | ((uint64_t)xhci_read32 (ir_base, XHCI_ERDP + 4) << 32); xhci_write32 (ir_base, XHCI_ERDP, (uint32_t)erdp | (1 << 3)); xhci_poll_events (xhci); spin_unlock (&xhci->device->lock, fd); } void xhci_send_cmd (struct xhci* xhci, uint64_t param, uint32_t status, uint32_t ctrl) { uint64_t fd; spin_lock (&xhci->device->lock, &fd); if (xhci->irqs_support) { if (xhci->cmd_ring_idx == (xhci->cmd_ring_size - 1)) { struct xhci_trb* link = &xhci->cmd_ring[xhci->cmd_ring_idx]; link->param = xhci->cmd_ring_phys; link->status = 0; link->ctrl = (6 << 10) | (1 << 1) | xhci->cmd_cycle_bit; xhci->cmd_ring_idx = 0; xhci->cmd_cycle_bit ^= 1; } struct xhci_trb* trb = &xhci->cmd_ring[xhci->cmd_ring_idx]; trb->param = param; trb->status = status; trb->ctrl = (ctrl & ~0x1) | xhci->cmd_cycle_bit; xhci->cmd_ring_idx++; atomic_store (&xhci->pending, true); xhci_write32 (xhci->xhci_doorbell_base, 0, 0); spin_unlock (&xhci->device->lock, fd); while (atomic_load (&xhci->pending)) spin_lock_relax (); spin_lock (&xhci->device->lock, &fd); } else { while (atomic_load (&xhci->pending)) { xhci_poll_events (xhci); spin_lock_relax (); } } spin_unlock (&xhci->device->lock, fd); } DEFINE_DEVICE_INIT (xhci_init) { struct limine_hhdm_response* hhdm = limine_hhdm_request.response; struct xhci* xhci = malloc (sizeof (*xhci)); if (xhci == NULL) return false; struct xhci_init* init = arg; memset (xhci, 0, sizeof (*xhci)); xhci->device = device; xhci->xhci_mmio_base = init->xhci_mmio_base; xhci->irqs_support = init->irqs_support; xhci->irq = init->irq; device->udata = xhci; uint8_t cap_length = xhci_read8 (xhci->xhci_mmio_base, XHCI_CAPLENGTH); xhci->xhci_oper_base = xhci->xhci_mmio_base + cap_length; uint32_t rtsoff = xhci_read32 (xhci->xhci_mmio_base, XHCI_RTSOFF); xhci->xhci_runtime_base = xhci->xhci_mmio_base + rtsoff; uint32_t dboff = xhci_read32 (xhci->xhci_mmio_base, XHCI_DBOFF); xhci->xhci_doorbell_base = xhci->xhci_mmio_base + dboff; DEBUG ("starting init sequence\n"); /* assert CNR is 0 */ while (xhci_read32 (xhci->xhci_oper_base, XHCI_USBSTS) & (1 << 11)) spin_lock_relax (); /* STOP */ xhci_write32 (xhci->xhci_oper_base, XHCI_USBCMD, 0); /* wait for HCH bit */ int timeout = 100000; while (!(xhci_read32 (xhci->xhci_oper_base, XHCI_USBSTS) & (1 << 12))) { if (--timeout == 0) break; spin_lock_relax (); } /* RESET */ xhci_write32 (xhci->xhci_oper_base, XHCI_USBCMD, (1 << 1)); while (xhci_read32 (xhci->xhci_oper_base, XHCI_USBCMD) & (1 << 1)) spin_lock_relax (); /* Stall while controller not ready */ while (xhci_read32 (xhci->xhci_oper_base, XHCI_USBSTS) & (1 << 11)) spin_lock_relax (); DEBUG ("XHCI init done\n"); uint32_t hcsparams1 = xhci_read32 (xhci->xhci_mmio_base, XHCI_HCSPARAMS1); uint8_t max_slots = (uint8_t)(hcsparams1 & 0xFF); /* enable slots */ uint32_t config = xhci_read32 (xhci->xhci_oper_base, XHCI_CONFIG); xhci_write32 (xhci->xhci_oper_base, XHCI_CONFIG, (config & ~0xFF) | max_slots); DEBUG ("enabled %u slots\n", max_slots); uint32_t hcsparams2 = xhci_read32 (xhci->xhci_mmio_base, XHCI_HCSPARAMS2); xhci->max_scratchpad = (hcsparams2 >> 27) & 0x1F; uintptr_t dcbaa_phys = pmm_alloc (1); xhci->xhci_dcbaa_phys = dcbaa_phys; xhci->xhci_dcbaa = (uintptr_t*)(dcbaa_phys + (uintptr_t)hhdm->offset); memset (xhci->xhci_dcbaa, 0, PAGE_SIZE); if (xhci->max_scratchpad > 0) { uintptr_t dev_array_phys = pmm_alloc (1); uintptr_t* dev_array = (uintptr_t*)(dev_array_phys + (uintptr_t)hhdm->offset); memset (dev_array, 0, PAGE_SIZE); for (size_t i = 0; i < xhci->max_scratchpad; i++) dev_array[i] = pmm_alloc (1); xhci->xhci_dcbaa[0] = dev_array_phys; } xhci_write32 (xhci->xhci_oper_base, XHCI_DCBAAP, (uint32_t)xhci->xhci_dcbaa_phys); xhci_write32 (xhci->xhci_oper_base, XHCI_DCBAAP + 4, (uint32_t)(xhci->xhci_dcbaa_phys >> 32)); xhci->cmd_ring_phys = pmm_alloc (1); xhci->cmd_ring = (struct xhci_trb*)(xhci->cmd_ring_phys + (uintptr_t)hhdm->offset); xhci->cmd_ring_size = PAGE_SIZE / sizeof (struct xhci_trb); xhci->cmd_ring_idx = 0; xhci->cmd_cycle_bit = 1; memset (xhci->cmd_ring, 0, PAGE_SIZE); uint64_t crcr = xhci->cmd_ring_phys | xhci->cmd_cycle_bit; xhci_write32 (xhci->xhci_oper_base, XHCI_CRCR, (uint32_t)crcr); xhci_write32 (xhci->xhci_oper_base, XHCI_CRCR + 4, (uint32_t)(crcr >> 32)); xhci->event_ring_phys = pmm_alloc (1); xhci->event_ring = (struct xhci_trb*)(xhci->event_ring_phys + (uintptr_t)hhdm->offset); xhci->event_ring_size = PAGE_SIZE / sizeof (struct xhci_trb); xhci->event_ring_idx = 0; xhci->event_cycle_bit = 1; memset (xhci->event_ring, 0, PAGE_SIZE); xhci->erst_phys = pmm_alloc (1); xhci->erst = (struct xhci_erst_entry*)(xhci->erst_phys + (uintptr_t)hhdm->offset); xhci->erst[0].ptr = xhci->event_ring_phys; xhci->erst[0].size = xhci->event_ring_size; xhci->erst[0]._rsvd = 0; uintptr_t ir_base = xhci->xhci_runtime_base + 0x20; xhci_write32 (ir_base, XHCI_ERSTSZ, 1); xhci_write32 (ir_base, XHCI_ERSTBA, (uint32_t)xhci->erst_phys); xhci_write32 (ir_base, XHCI_ERSTBA + 4, (uint32_t)(xhci->erst_phys >> 32)); xhci_write32 (ir_base, XHCI_ERDP, (uint32_t)xhci->event_ring_phys | (1 << 3)); xhci_write32 (ir_base, XHCI_ERDP + 4, (uint32_t)(xhci->event_ring_phys >> 32)); if (xhci->irqs_support) { /* enable interrupter */ irq_attach (&xhci_irq, xhci, xhci->irq); xhci_write32 (ir_base, XHCI_IMAN, xhci_read32 (ir_base, XHCI_IMAN) | 0x02); } uint32_t usbcmd = xhci_read32 (xhci->xhci_oper_base, XHCI_USBCMD); xhci_write32 (xhci->xhci_oper_base, XHCI_USBCMD, usbcmd | 0x01 | (1 << 2)); xhci_send_cmd (xhci, 0, 0, (23 << 10)); return true; } DEFINE_DEVICE_FINI (xhci_fini) { struct limine_hhdm_response* hhdm = limine_hhdm_request.response; struct xhci* xhci = device->udata; if (xhci->max_scratchpad > 0) { uintptr_t dev_array_phys = xhci->xhci_dcbaa[0]; uintptr_t* dev_array = (uintptr_t*)(dev_array_phys + (uintptr_t)hhdm->offset); for (size_t i = 0; i < xhci->max_scratchpad; i++) { if (dev_array[i] != 0) pmm_free (dev_array[i], 1); } pmm_free (dev_array_phys, 1); } pmm_free (xhci->xhci_dcbaa_phys, 1); pmm_free (xhci->cmd_ring_phys, 1); pmm_free (xhci->event_ring_phys, 1); pmm_free (xhci->erst_phys, 1); irq_detach (xhci->irq); free (xhci); }