#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* REF: * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf */ /* clang-format off */ /* capability registers */ #define XHCI_CAPLENGTH 0x00 #define XHCI_RSVD 0x01 #define XHCI_HCIVERSION 0x02 #define XHCI_HCSPARAMS1 0x04 #define XHCI_HCSPARAMS2 0x08 #define XHCI_HCSPARAMS3 0x0C #define XHCI_HCCPARAMS1 0x10 #define XHCI_DBOFF 0x14 #define XHCI_RTSOFF 0x18 #define XHCI_HCCPARAMS2 0x1C /* operational registers */ #define XHCI_USBCMD 0x00 #define XHCI_USBSTS 0x04 #define XHCI_PAGESIZE 0x08 #define XHCI_DNCTRL 0x14 #define XHCI_CRCR 0x18 #define XHCI_DCBAAP 0x30 #define XHCI_CONFIG 0x38 /* port registers */ #define XHCI_PORTSC 0x00 #define XHCI_PORTPMSC 0x04 #define XHCI_PORTLI 0x08 /* runtime registers */ #define XHCI_MFINDEX 0x00 /* + IRQ sets (0x20) */ #define XHCI_IMAN 0x00 #define XHCI_IMOD 0x04 #define XHCI_ERSTSZ 0x08 #define XHCI_ERSTBA 0x10 #define XHCI_ERDP 0x18 /* event types */ #define XHCI_TRB_NORMAL 1 #define XHCI_TRB_SETUP_STAGE 2 #define XHCI_TRB_DATA_STAGE 3 #define XHCI_TRB_STATUS_STAGE 4 #define XHCI_TRB_ISOCH 5 #define XHCI_TRB_LINK 6 #define XHCI_TRB_EVENT_DATA 7 #define XHCI_TRB_NOOP 8 #define XHCI_TRB_SLOT_ENAB_CMD 9 #define XHCI_TRB_SLOT_DISB_CMD 10 #define XHCI_TRB_ADDR_DEV_CMD 11 #define XHCI_TRB_CFG_ENDP_CMD 12 #define XHCI_TRB_EVAL_CTX_CMD 13 #define XHCI_TRB_RESET_ENDP_CMD 14 #define XHCI_TRB_STOP_ENDP_CMD 15 #define XHCI_TRB_SET_DRDQP_CMD 16 #define XHCI_TRB_RESET_DEV_CMD 17 #define XHCI_TRB_FORCE_EVT_CMD 18 #define XHCI_TRB_NEGO_BANDW_CMD 19 #define XHCI_TRB_SET_LTV_CMD 20 #define XHCI_TRB_PORT_BANDW_CMD 21 #define XHCI_TRB_FORCE_HEADER 22 #define XHCI_TRB_NOOP_CMD 23 #define XHCI_TRB_GET_EXTPRP_CMD 24 #define XHCI_TRB_SET_EXTPRP_CMD 25 #define XHCI_TRB_TRANSFER_EVENT 32 #define XHCI_TRB_CMD_CMPL_EVENT 33 #define XHCI_TRB_PORT_STS_CHNG 34 #define XHCI_TRB_BANDW_RQ_EVENT 35 #define XHCI_TRB_DOORBELL_EVENT 36 #define XHCI_TRB_HOST_CTRL_EVNT 37 #define XHCI_TRB_DEV_NOTIF_EVNT 38 #define XHCI_TRB_MFINDEX_WRAP 39 /* clang-format on */ static void xhci_write8 (uintptr_t base, uint32_t reg, uint8_t value) { *(volatile uint8_t*)(base + reg) = value; } static void xhci_write16 (uintptr_t base, uint32_t reg, uint16_t value) { *(volatile uint16_t*)(base + reg) = value; } static void xhci_write32 (uintptr_t base, uint32_t reg, uint32_t value) { *(volatile uint32_t*)(base + reg) = value; } static uint8_t xhci_read8 (uintptr_t base, uint32_t reg) { return *(volatile uint8_t*)(base + reg); } static uint16_t xhci_read16 (uintptr_t base, uint32_t reg) { return *(volatile uint16_t*)(base + reg); } static uint32_t xhci_read32 (uintptr_t base, uint32_t reg) { return *(volatile uint32_t*)(base + reg); } static void xhci_event_dispatch (struct xhci* xhci, struct xhci_trb* event, uint8_t type) { switch (type) { case XHCI_TRB_CMD_CMPL_EVENT: { uint8_t cmpl_code = (event->status >> 24) & 0xFF; uint8_t slot_id = (event->ctrl >> 24) & 0xFF; DEBUG ("cmd completion: code=%u,slot=%u\n", cmpl_code, slot_id); } break; default: DEBUG ("Unhandled event type %u at %u\n", type, xhci->event_ring_idx); break; } } static void xhci_poll_events (struct xhci* xhci) { uintptr_t ir_base = xhci->xhci_runtime_base + 0x20; bool serviced = false; for (;;) { struct xhci_trb* event = &xhci->event_ring[xhci->event_ring_idx]; if ((event->ctrl & 0x1) != xhci->event_cycle_bit) { break; } serviced = true; uint8_t type = (event->ctrl >> 10) & 0x3F; xhci_event_dispatch (xhci, event, type); xhci->event_ring_idx++; if (xhci->event_ring_idx >= xhci->event_ring_size) { xhci->event_ring_idx = 0; xhci->event_cycle_bit ^= 1; } } if (serviced) { uintptr_t dequeue_ptr = xhci->event_ring_phys + (xhci->event_ring_idx * sizeof (struct xhci_trb)); xhci_write32 (ir_base, XHCI_ERDP, (uint32_t)dequeue_ptr | (1 << 3)); xhci_write32 (ir_base, XHCI_ERDP + 4, (uint32_t)(dequeue_ptr >> 32)); atomic_store (&xhci->pending, false); } } static void xhci_irq (void* arg, void* regs, bool user, struct reschedule_ctx* rctx) { (void)user, (void)regs, (void)rctx; uint64_t fd; struct xhci* xhci = arg; spin_lock (&xhci->device->lock, &fd); uintptr_t ir_base = xhci->xhci_runtime_base + 0x20; /* ack */ xhci_write32 (ir_base, XHCI_IMAN, xhci_read32 (ir_base, XHCI_IMAN) | (1 << 0)); xhci_write32 (xhci->xhci_oper_base, XHCI_USBSTS, (1 << 3)); xhci_poll_events (xhci); spin_unlock (&xhci->device->lock, fd); } void xhci_send_cmd (struct xhci* xhci, uint64_t param, uint32_t status, uint32_t ctrl) { uint64_t fd; spin_lock (&xhci->device->lock, &fd); if (xhci->irqs_support) { if (xhci->cmd_ring_idx == (xhci->cmd_ring_size - 1)) { struct xhci_trb* link = &xhci->cmd_ring[xhci->cmd_ring_idx]; link->param = xhci->cmd_ring_phys; link->status = 0; link->ctrl = (XHCI_TRB_LINK << 10) | (1 << 1) | xhci->cmd_cycle_bit; xhci->cmd_ring_idx = 0; xhci->cmd_cycle_bit ^= 1; } struct xhci_trb* trb = &xhci->cmd_ring[xhci->cmd_ring_idx]; trb->param = param; trb->status = status; trb->ctrl = (ctrl & ~0x1) | xhci->cmd_cycle_bit; xhci->cmd_ring_idx++; atomic_store (&xhci->pending, true); xhci_write32 (xhci->xhci_doorbell_base, 0, 0); spin_unlock (&xhci->device->lock, fd); int timeout = 100; while (atomic_load (&xhci->pending) && --timeout > 0) stall_ms (1); spin_lock (&xhci->device->lock, &fd); if (timeout == 0) DEBUG ("timed out\n"); } else { while (atomic_load (&xhci->pending)) { xhci_poll_events (xhci); spin_lock_relax (); } } spin_unlock (&xhci->device->lock, fd); } /* Make or break on real hardware. We need to take over ownership from the BIOS. */ static void xhci_bios_handover (struct xhci* xhci) { uint32_t hccparams1 = xhci_read32 (xhci->xhci_mmio_base, XHCI_HCCPARAMS1); uint32_t ext_offset = (hccparams1 >> 16) << 2; if (ext_offset == 0) return; while (ext_offset) { uintptr_t cap_ptr = xhci->xhci_mmio_base + ext_offset; uint32_t cap = xhci_read32 (cap_ptr, 0); uint8_t cap_id = cap & 0xFF; if (cap_id == 1) { DEBUG ("Found USB Legacy Support at offset 0x%x\n", ext_offset); if (cap & (1 << 16)) { DEBUG ("BIOS owns XHCI, requesting handover!\n"); xhci_write8 (cap_ptr, 3, 1); /* Wait for BIOS Semaphore owned bit to be cleared */ int timeout = 1000; while (--timeout > 0) { uint32_t val = xhci_read32 (cap_ptr, 0); if (!(val & (1 << 16)) && (val & (1 << 24))) break; stall_ms (100); } DEBUG ("XHCI Handover OK\n"); } break; } uint8_t next = (cap >> 8) & 0xFF; if (!next) break; ext_offset += (next << 2); } } DEFINE_DEVICE_INIT (xhci_init) { struct limine_hhdm_response* hhdm = limine_hhdm_request.response; struct xhci* xhci = malloc (sizeof (*xhci)); if (xhci == NULL) return false; struct xhci_init* init = arg; memset (xhci, 0, sizeof (*xhci)); xhci->device = device; xhci->xhci_mmio_base = init->xhci_mmio_base; xhci->irqs_support = init->irqs_support; xhci->irq = init->irq; device->udata = xhci; uint32_t usbcmd, config, cap_length; cap_length = xhci_read8 (xhci->xhci_mmio_base, XHCI_CAPLENGTH); xhci->xhci_oper_base = xhci->xhci_mmio_base + cap_length; uint32_t rtsoff = xhci_read32 (xhci->xhci_mmio_base, XHCI_RTSOFF); xhci->xhci_runtime_base = xhci->xhci_mmio_base + rtsoff; uint32_t dboff = xhci_read32 (xhci->xhci_mmio_base, XHCI_DBOFF); xhci->xhci_doorbell_base = xhci->xhci_mmio_base + dboff; uint32_t hcsparams2 = xhci_read32 (xhci->xhci_mmio_base, XHCI_HCSPARAMS2); xhci->max_scratchpad = (((hcsparams2 >> 21) & 0x1F) << 5) | ((hcsparams2 >> 27) & 0x1F); DEBUG ("starting init sequence\n"); /* stop running / clear Run/Stop bit */ usbcmd = xhci_read32 (xhci->xhci_oper_base, XHCI_USBCMD); usbcmd &= ~(1 << 0); xhci_write32 (xhci->xhci_oper_base, XHCI_USBCMD, usbcmd); stall_ms (1000); xhci_bios_handover (xhci); /* reset controller */ usbcmd = xhci_read32 (xhci->xhci_oper_base, XHCI_USBCMD); usbcmd |= (1 << 1); xhci_write32 (xhci->xhci_oper_base, XHCI_USBCMD, usbcmd); while (xhci_read32 (xhci->xhci_oper_base, XHCI_USBSTS) & (1 << 11)) spin_lock_relax (); DEBUG ("controller reset\n"); xhci->max_slots = xhci_read32 (xhci->xhci_mmio_base, XHCI_HCSPARAMS1) & 0xFF; DEBUG ("max_slots=%u\n", xhci->max_slots); /* enable device notifications */ xhci_write32 (xhci->xhci_oper_base, XHCI_DNCTRL, 0xFFFF); /* enable slots */ config = xhci_read32 (xhci->xhci_oper_base, XHCI_CONFIG); xhci_write32 (xhci->xhci_oper_base, XHCI_CONFIG, (config & ~0xFF) | (xhci->max_slots & 0xFF)); /* Prepare DCBAA */ xhci->xhci_dcbaa_phys = pmm_alloc (1); xhci->xhci_dcbaa = (uintptr_t*)(xhci->xhci_dcbaa_phys + (uintptr_t)hhdm->offset); memset (xhci->xhci_dcbaa, 0, PAGE_SIZE); if (xhci->max_scratchpad > 0) { xhci->scratchpads_phys = pmm_alloc (1); xhci->scratchpads = (uintptr_t*)(xhci->scratchpads_phys + (uintptr_t)hhdm->offset); memset (xhci->scratchpads, 0, PAGE_SIZE); for (size_t sp = 0; sp < xhci->max_scratchpad; sp++) { xhci->scratchpads[sp] = pmm_alloc (1); } xhci->xhci_dcbaa[0] = xhci->scratchpads_phys; } xhci_write32 (xhci->xhci_oper_base, XHCI_DCBAAP, (uint32_t)xhci->xhci_dcbaa_phys); xhci_write32 (xhci->xhci_oper_base, XHCI_DCBAAP + 4, (uint32_t)(xhci->xhci_dcbaa_phys >> 32)); xhci->cmd_ring_phys = pmm_alloc (1); xhci->cmd_ring = (struct xhci_trb*)(xhci->cmd_ring_phys + (uintptr_t)hhdm->offset); memset (xhci->cmd_ring, 0, PAGE_SIZE); xhci->cmd_ring_size = PAGE_SIZE / sizeof (struct xhci_trb); xhci->cmd_ring_idx = 0; xhci->cmd_cycle_bit = 1; uint64_t crcr = xhci->cmd_ring_phys | xhci->cmd_cycle_bit; xhci_write32 (xhci->xhci_oper_base, XHCI_CRCR, (uint32_t)crcr); xhci_write32 (xhci->xhci_oper_base, XHCI_CRCR + 4, (uint32_t)(crcr >> 32)); xhci->event_ring_phys = pmm_alloc (1); xhci->event_ring = (struct xhci_trb*)(xhci->event_ring_phys + (uintptr_t)hhdm->offset); memset (xhci->event_ring, 0, PAGE_SIZE); xhci->event_ring_size = PAGE_SIZE / sizeof (struct xhci_trb); xhci->event_ring_idx = 0; xhci->event_cycle_bit = 1; xhci->erst_phys = pmm_alloc (1); xhci->erst = (struct xhci_erst_entry*)(xhci->erst_phys + (uintptr_t)hhdm->offset); memset (xhci->erst, 0, PAGE_SIZE); xhci->erst[0].ptr = xhci->event_ring_phys; xhci->erst[0].size = xhci->event_ring_size; xhci->erst[0]._rsvd = 0; uintptr_t ir_base = xhci->xhci_runtime_base + 0x20; xhci_write32 (ir_base, XHCI_ERSTSZ, 1); xhci_write32 (ir_base, XHCI_ERSTBA, (uint32_t)xhci->erst_phys); xhci_write32 (ir_base, XHCI_ERSTBA + 4, (uint32_t)(xhci->erst_phys >> 32)); xhci_write32 (ir_base, XHCI_ERDP, (uint32_t)xhci->event_ring_phys | (1 << 3)); xhci_write32 (ir_base, XHCI_ERDP + 4, (uint32_t)(xhci->event_ring_phys >> 32)); if (xhci->irqs_support) { /* enable interrupter */ irq_attach (&xhci_irq, xhci, xhci->irq); xhci_write32 (ir_base, XHCI_IMAN, xhci_read32 (ir_base, XHCI_IMAN) | (1 << 1)); } usbcmd = xhci_read32 (xhci->xhci_oper_base, XHCI_USBCMD); xhci_write32 (xhci->xhci_oper_base, XHCI_USBCMD, usbcmd | (1 << 0) | (1 << 2)); xhci_send_cmd (xhci, 0, 0, XHCI_TRB_SLOT_ENAB_CMD << 10); xhci_send_cmd (xhci, 0, 0, XHCI_TRB_NOOP_CMD << 10); return true; } DEFINE_DEVICE_FINI (xhci_fini) { struct limine_hhdm_response* hhdm = limine_hhdm_request.response; struct xhci* xhci = device->udata; if (xhci->max_scratchpad > 0) { uintptr_t scratchpads_phys = xhci->xhci_dcbaa[0]; uintptr_t* scratchpads = (uintptr_t*)(scratchpads_phys + (uintptr_t)hhdm->offset); for (size_t i = 0; i < xhci->max_scratchpad; i++) { if (scratchpads[i] != 0) pmm_free (scratchpads[i], 1); } pmm_free (scratchpads_phys, 1); } pmm_free (xhci->xhci_dcbaa_phys, 1); pmm_free (xhci->cmd_ring_phys, 1); pmm_free (xhci->event_ring_phys, 1); pmm_free (xhci->erst_phys, 1); irq_detach (xhci->irq); free (xhci); }