#include #include #include #include #include #include #include #include #include #include #include #include /* Present flag */ #define AMD64_PG_PRESENT (1 << 0) /* Writable flag */ #define AMD64_PG_RW (1 << 1) /* User-accessible flag */ #define AMD64_PG_USER (1 << 2) /* Auxilary struct for page directory walking */ struct pg_index { uint16_t pml4, pml3, pml2, pml1; } PACKED; /* Kernel page directory */ static struct pd kernel_pd = {.lock = SPIN_LOCK_INIT}; /* Lock needed to sync between map/unmap operations and TLB shootdown */ static spin_lock_t mm_lock = SPIN_LOCK_INIT; /* Get current value of CR3 register */ static uintptr_t amd64_current_cr3 (void) { uintptr_t cr3; __asm__ volatile ("movq %%cr3, %0" : "=r"(cr3)::"memory"); return cr3; } /* Load kernel CR3 as current CR3 */ void amd64_load_kernel_cr3 (void) { __asm__ volatile ("movq %0, %%cr3" ::"r"(kernel_pd.cr3_paddr) : "memory"); } /* Extract PML info from virtual address */ static struct pg_index amd64_mm_page_index (uint64_t vaddr) { struct pg_index ret; ret.pml4 = ((vaddr >> 39) & 0x1FF); ret.pml3 = ((vaddr >> 30) & 0x1FF); ret.pml2 = ((vaddr >> 21) & 0x1FF); ret.pml1 = ((vaddr >> 12) & 0x1FF); return ret; } /* Walk paging tables and allocate necessary structures along the way */ static uint64_t* amd64_mm_next_table (uint64_t* table, uint64_t entry_idx, bool alloc) { uint64_t entry = table[entry_idx]; physaddr_t paddr; struct limine_hhdm_response* hhdm = limine_hhdm_request.response; if (entry & AMD64_PG_PRESENT) paddr = entry & ~0xFFFULL; else { if (!alloc) return NULL; paddr = pmm_alloc (1); if (paddr == PMM_ALLOC_ERR) return NULL; memset ((void*)((uintptr_t)hhdm->offset + (uintptr_t)paddr), 0, PAGE_SIZE); table[entry_idx] = paddr | AMD64_PG_PRESENT | AMD64_PG_RW | AMD64_PG_USER; } return (uint64_t*)((uintptr_t)hhdm->offset + (uintptr_t)paddr); } static bool amd64_mm_is_table_empty (uint64_t* table) { for (size_t i = 0; i < 512; i++) { if (table[i] & AMD64_PG_PRESENT) return false; } return true; } /* Convert generic memory management subsystem flags into AMD64-specific flags */ static uint64_t amd64_mm_resolve_flags (uint32_t generic) { uint64_t flags = 0; flags |= ((generic & MM_PG_PRESENT) ? AMD64_PG_PRESENT : 0); flags |= ((generic & MM_PG_RW) ? AMD64_PG_RW : 0); flags |= ((generic & MM_PG_USER) ? AMD64_PG_USER : 0); return flags; } /* Reload the current CR3 value ON A LOCAL CPU */ static void amd64_reload_cr3 (void) { uint64_t cr3; __asm__ volatile ("movq %%cr3, %0; movq %0, %%cr3" : "=r"(cr3)::"memory"); } /* Map physical address to virtual address with flags. TLB needs to be flushed afterwards. */ void mm_map_page (struct pd* pd, uintptr_t paddr, uintptr_t vaddr, uint32_t flags) { spin_lock (&mm_lock); struct limine_hhdm_response* hhdm = limine_hhdm_request.response; bool do_reload = false; if (flags & MM_PD_LOCK) spin_lock (&pd->lock); uint64_t amd64_flags = amd64_mm_resolve_flags (flags); uint64_t* pml4 = (uint64_t*)(pd->cr3_paddr + (uintptr_t)hhdm->offset); struct pg_index pg_index = amd64_mm_page_index (vaddr); uint64_t* pml3 = amd64_mm_next_table (pml4, pg_index.pml4, true); if (pml3 == NULL) goto done; uint64_t* pml2 = amd64_mm_next_table (pml3, pg_index.pml3, true); if (pml2 == NULL) goto done; uint64_t* pml1 = amd64_mm_next_table (pml2, pg_index.pml2, true); if (pml1 == NULL) goto done; uint64_t* pte = &pml1[pg_index.pml1]; *pte = ((paddr & ~0xFFFULL) | (amd64_flags & 0x7ULL)); do_reload = true; done: if (do_reload && (flags & MM_PD_RELOAD)) amd64_reload_cr3 (); if (flags & MM_PD_LOCK) spin_unlock (&pd->lock); spin_unlock (&mm_lock); } /* Map a page into kernel page directory */ void mm_map_kernel_page (uintptr_t paddr, uintptr_t vaddr, uint32_t flags) { mm_map_page (&kernel_pd, paddr, vaddr, flags); } /* Unmap a virtual address. TLB needs to be flushed afterwards */ void mm_unmap_page (struct pd* pd, uintptr_t vaddr, uint32_t flags) { spin_lock (&mm_lock); struct limine_hhdm_response* hhdm = limine_hhdm_request.response; bool do_reload = false; if (flags & MM_PD_LOCK) spin_lock (&pd->lock); uint64_t* pml4 = (uint64_t*)(pd->cr3_paddr + (uintptr_t)hhdm->offset); struct pg_index pg_index = amd64_mm_page_index (vaddr); uint64_t* pml3 = amd64_mm_next_table (pml4, pg_index.pml4, false); if (pml3 == NULL) goto done; uint64_t* pml2 = amd64_mm_next_table (pml3, pg_index.pml3, false); if (pml2 == NULL) goto done; uint64_t* pml1 = amd64_mm_next_table (pml2, pg_index.pml2, false); if (pml1 == NULL) goto done; uint64_t* pte = &pml1[pg_index.pml1]; if ((*pte) & AMD64_PG_PRESENT) { *pte = 0; do_reload = true; } if (amd64_mm_is_table_empty (pml1)) { uintptr_t pml1_phys = pml2[pg_index.pml2] & ~0xFFFULL; pmm_free (pml1_phys, 1); pml2[pg_index.pml2] = 0; if (amd64_mm_is_table_empty (pml2)) { uintptr_t pml2_phys = pml3[pg_index.pml3] & ~0xFFFULL; pmm_free (pml2_phys, 1); pml3[pg_index.pml3] = 0; if (amd64_mm_is_table_empty (pml3)) { uintptr_t pml3_phys = pml4[pg_index.pml4] & ~0xFFFULL; pmm_free (pml3_phys, 1); pml4[pg_index.pml4] = 0; } } } done: if (do_reload && (flags & MM_PD_RELOAD)) amd64_reload_cr3 (); if (flags & MM_PD_LOCK) spin_unlock (&pd->lock); spin_unlock (&mm_lock); } /* Unmap a page from kernel page directory */ void mm_unmap_kernel_page (uintptr_t vaddr, uint32_t flags) { mm_unmap_page (&kernel_pd, vaddr, flags); } /* Lock kernel page directory */ void mm_lock_kernel (void) { spin_lock (&kernel_pd.lock); } /* Unlock kernel page directory */ void mm_unlock_kernel (void) { spin_unlock (&kernel_pd.lock); } /* Allocate a userspace-ready page directory */ uintptr_t mm_alloc_user_pd_phys (void) { struct limine_hhdm_response* hhdm = limine_hhdm_request.response; physaddr_t cr3 = pmm_alloc (1); if (cr3 == PMM_ALLOC_ERR) return 0; uint8_t* vu_cr3 = (uint8_t*)((uintptr_t)hhdm->offset + cr3); memset ((void*)vu_cr3, 0, PAGE_SIZE / 2); uint8_t* vk_cr3 = (uint8_t*)((uintptr_t)hhdm->offset + (uintptr_t)kernel_pd.cr3_paddr); memcpy (&vu_cr3[PAGE_SIZE / 2], &vk_cr3[PAGE_SIZE / 2], PAGE_SIZE / 2); return cr3; } /* Reload after map/unmap operation was performed. This function does the TLB shootdown. */ void mm_reload (void) { spin_lock (&mm_lock); struct limine_mp_response* mp = limine_mp_request.response; for (size_t i = 0; i < mp->cpu_count; i++) { amd64_lapic_ipi (mp->cpus[i]->lapic_id, TLB_SHOOTDOWN); } spin_unlock (&mm_lock); } /* TLB shootdown IRQ handler */ static void amd64_tlb_shootdown_irq (void* arg, void* regs) { (void)arg, (void)regs; amd64_reload_cr3 (); DEBUG ("cpu %u TLB shootdown\n", thiscpu->id); } /* Continue initializing memory management subsystem for AMD64 after the essential parts were * initialized */ void mm_init2 (void) { irq_attach (&amd64_tlb_shootdown_irq, NULL, TLB_SHOOTDOWN, IRQ_INTERRUPT_SAFE); } /* Initialize essentials for the AMD64 memory management subsystem */ void mm_init (void) { kernel_pd.cr3_paddr = amd64_current_cr3 (); }