mop3/kernel/amd64/mm.c

#include <amd64/apic.h>
#include <amd64/intr_defs.h>
#include <aux/compiler.h>
#include <irq/irq.h>
#include <libk/std.h>
#include <libk/string.h>
#include <limine/requests.h>
#include <mm/pmm.h>
#include <sync/spin_lock.h>
#include <sys/debug.h>
#include <sys/mm.h>
#include <sys/smp.h>

/* Present flag */
#define AMD64_PG_PRESENT (1 << 0)
/* Writable flag */
#define AMD64_PG_RW (1 << 1)
/* User-accessible flag */
#define AMD64_PG_USER (1 << 2)

/* Auxilary struct for page directory walking */
struct pg_index {
  uint16_t pml4, pml3, pml2, pml1;
} PACKED;

/* Kernel page directory */
static struct pd kernel_pd = {.lock = SPIN_LOCK_INIT};
/* Lock needed to sync between map/unmap operations and TLB shootdown */
static spin_lock_t mm_lock = SPIN_LOCK_INIT;

/* Get current value of CR3 register */
static uintptr_t amd64_current_cr3 (void) {
  uintptr_t cr3;
  __asm__ volatile ("movq %%cr3, %0" : "=r"(cr3)::"memory");
  return cr3;
}

/* Load kernel CR3 as current CR3 */
void amd64_load_kernel_cr3 (void) {
  __asm__ volatile ("movq %0, %%cr3" ::"r"(kernel_pd.cr3_paddr) : "memory");
}

/* Extract PML info from virtual address */
static struct pg_index amd64_mm_page_index (uint64_t vaddr) {
  struct pg_index ret;

  ret.pml4 = ((vaddr >> 39) & 0x1FF);
  ret.pml3 = ((vaddr >> 30) & 0x1FF);
  ret.pml2 = ((vaddr >> 21) & 0x1FF);
  ret.pml1 = ((vaddr >> 12) & 0x1FF);

  return ret;
}

/* Walk paging tables and allocate necessary structures along the way */
static uint64_t* amd64_mm_next_table (uint64_t* table, uint64_t entry_idx, bool alloc) {
  uint64_t entry = table[entry_idx];
  physaddr_t paddr;

  struct limine_hhdm_response* hhdm = limine_hhdm_request.response;

  if (entry & AMD64_PG_PRESENT)
    paddr = entry & ~0xFFFULL;
  else {
    if (!alloc)
      return NULL;

    paddr = pmm_alloc (1);

    if (paddr == PMM_ALLOC_ERR)
      return NULL;

    memset ((void*)((uintptr_t)hhdm->offset + (uintptr_t)paddr), 0, PAGE_SIZE);
    table[entry_idx] = paddr | AMD64_PG_PRESENT | AMD64_PG_RW | AMD64_PG_USER;
  }

  return (uint64_t*)((uintptr_t)hhdm->offset + (uintptr_t)paddr);
}

static bool amd64_mm_is_table_empty (uint64_t* table) {
  for (size_t i = 0; i < 512; i++) {
    if (table[i] & AMD64_PG_PRESENT)
      return false;
  }
  return true;
}

/* Convert generic memory management subsystem flags into AMD64-specific flags */
static uint64_t amd64_mm_resolve_flags (uint32_t generic) {
  uint64_t flags = 0;

  flags |= ((generic & MM_PG_PRESENT) ? AMD64_PG_PRESENT : 0);
  flags |= ((generic & MM_PG_RW) ? AMD64_PG_RW : 0);
  flags |= ((generic & MM_PG_USER) ? AMD64_PG_USER : 0);

  return flags;
}

/* Reload the current CR3 value ON A LOCAL CPU */
static void amd64_reload_cr3 (void) {
  uint64_t cr3;
  __asm__ volatile ("movq %%cr3, %0; movq %0, %%cr3" : "=r"(cr3)::"memory");
}

/* Map physical address to virtual address with flags. TLB needs to be flushed afterwards. */
void mm_map_page (struct pd* pd, uintptr_t paddr, uintptr_t vaddr, uint32_t flags) {
  spin_lock (&mm_lock);

  struct limine_hhdm_response* hhdm = limine_hhdm_request.response;
  bool do_reload = false;

  if (flags & MM_PD_LOCK)
    spin_lock (&pd->lock);

  uint64_t amd64_flags = amd64_mm_resolve_flags (flags);

  uint64_t* pml4 = (uint64_t*)(pd->cr3_paddr + (uintptr_t)hhdm->offset);
  struct pg_index pg_index = amd64_mm_page_index (vaddr);

  uint64_t* pml3 = amd64_mm_next_table (pml4, pg_index.pml4, true);
  if (pml3 == NULL)
    goto done;

  uint64_t* pml2 = amd64_mm_next_table (pml3, pg_index.pml3, true);
  if (pml2 == NULL)
    goto done;

  uint64_t* pml1 = amd64_mm_next_table (pml2, pg_index.pml2, true);
  if (pml1 == NULL)
    goto done;

  uint64_t* pte = &pml1[pg_index.pml1];

  *pte = ((paddr & ~0xFFFULL) | (amd64_flags & 0x7ULL));
  do_reload = true;

done:
  if (do_reload && (flags & MM_PD_RELOAD))
    amd64_reload_cr3 ();

  if (flags & MM_PD_LOCK)
    spin_unlock (&pd->lock);

  spin_unlock (&mm_lock);
}

/* Map a page into kernel page directory */
void mm_map_kernel_page (uintptr_t paddr, uintptr_t vaddr, uint32_t flags) {
  mm_map_page (&kernel_pd, paddr, vaddr, flags);
}

/* Unmap a virtual address. TLB needs to be flushed afterwards */
void mm_unmap_page (struct pd* pd, uintptr_t vaddr, uint32_t flags) {
  spin_lock (&mm_lock);

  struct limine_hhdm_response* hhdm = limine_hhdm_request.response;
  bool do_reload = false;

  if (flags & MM_PD_LOCK)
    spin_lock (&pd->lock);

  uint64_t* pml4 = (uint64_t*)(pd->cr3_paddr + (uintptr_t)hhdm->offset);
  struct pg_index pg_index = amd64_mm_page_index (vaddr);

  uint64_t* pml3 = amd64_mm_next_table (pml4, pg_index.pml4, false);
  if (pml3 == NULL)
    goto done;

  uint64_t* pml2 = amd64_mm_next_table (pml3, pg_index.pml3, false);
  if (pml2 == NULL)
    goto done;

  uint64_t* pml1 = amd64_mm_next_table (pml2, pg_index.pml2, false);
  if (pml1 == NULL)
    goto done;

  uint64_t* pte = &pml1[pg_index.pml1];

  if ((*pte) & AMD64_PG_PRESENT) {
    *pte = 0;
    do_reload = true;
  }

  if (amd64_mm_is_table_empty (pml1)) {
    uintptr_t pml1_phys = pml2[pg_index.pml2] & ~0xFFFULL;
    pmm_free (pml1_phys, 1);
    pml2[pg_index.pml2] = 0;

    if (amd64_mm_is_table_empty (pml2)) {
      uintptr_t pml2_phys = pml3[pg_index.pml3] & ~0xFFFULL;
      pmm_free (pml2_phys, 1);
      pml3[pg_index.pml3] = 0;

      if (amd64_mm_is_table_empty (pml3)) {
        uintptr_t pml3_phys = pml4[pg_index.pml4] & ~0xFFFULL;
        pmm_free (pml3_phys, 1);
        pml4[pg_index.pml4] = 0;
      }
    }
  }

done:
  if (do_reload && (flags & MM_PD_RELOAD))
    amd64_reload_cr3 ();

  if (flags & MM_PD_LOCK)
    spin_unlock (&pd->lock);

  spin_unlock (&mm_lock);
}

/* Unmap a page from kernel page directory */
void mm_unmap_kernel_page (uintptr_t vaddr, uint32_t flags) {
  mm_unmap_page (&kernel_pd, vaddr, flags);
}

/* Lock kernel page directory */
void mm_lock_kernel (void) { spin_lock (&kernel_pd.lock); }

/* Unlock kernel page directory */
void mm_unlock_kernel (void) { spin_unlock (&kernel_pd.lock); }

/* Allocate a userspace-ready page directory */
uintptr_t mm_alloc_user_pd_phys (void) {
  struct limine_hhdm_response* hhdm = limine_hhdm_request.response;

  physaddr_t cr3 = pmm_alloc (1);
  if (cr3 == PMM_ALLOC_ERR)
    return 0;

  uint8_t* vu_cr3 = (uint8_t*)((uintptr_t)hhdm->offset + cr3);
  memset ((void*)vu_cr3, 0, PAGE_SIZE / 2);

  uint8_t* vk_cr3 = (uint8_t*)((uintptr_t)hhdm->offset + (uintptr_t)kernel_pd.cr3_paddr);

  memcpy (&vu_cr3[PAGE_SIZE / 2], &vk_cr3[PAGE_SIZE / 2], PAGE_SIZE / 2);

  return cr3;
}

/* Reload after map/unmap operation was performed. This function does the TLB shootdown. */
void mm_reload (void) {
  spin_lock (&mm_lock);

  struct limine_mp_response* mp = limine_mp_request.response;

  for (size_t i = 0; i < mp->cpu_count; i++) {
    amd64_lapic_ipi (mp->cpus[i]->lapic_id, TLB_SHOOTDOWN);
  }

  spin_unlock (&mm_lock);
}

/* TLB shootdown IRQ handler */
static void amd64_tlb_shootdown_irq (void* arg, void* regs) {
  (void)arg, (void)regs;

  amd64_reload_cr3 ();
  DEBUG ("cpu %u TLB shootdown\n", thiscpu->id);
}

/* Continue initializing memory management subsystem for AMD64 after the essential parts were
 * initialized */
void mm_init2 (void) {
  irq_attach (&amd64_tlb_shootdown_irq, NULL, TLB_SHOOTDOWN, IRQ_INTERRUPT_SAFE);
}

/* Initialize essentials for the AMD64 memory management subsystem */
void mm_init (void) { kernel_pd.cr3_paddr = amd64_current_cr3 (); }