Spinlock save cpu flags

kernel/amd64/apic.c

@@ -57,19 +57,23 @@ static spin_lock_t lapic_calibration_lock = SPIN_LOCK_INIT;
 
 /* Read IOAPIC */
 static uint32_t ioapic_read (struct ioapic* ioapic, uint32_t reg) {
-  spin_lock (&ioapic->lock);
+  uint64_t fia;
+
+  spin_lock (&ioapic->lock, &fia);
   *(volatile uint32_t*)ioapic->mmio_base = reg;
   uint32_t ret = *(volatile uint32_t*)(ioapic->mmio_base + 0x10);
-  spin_unlock (&ioapic->lock);
+  spin_unlock (&ioapic->lock, fia);
   return ret;
 }
 
 /* Write IOAPIC */
 static void ioapic_write (struct ioapic* ioapic, uint32_t reg, uint32_t value) {
-  spin_lock (&ioapic->lock);
+  uint64_t fia;
+
+  spin_lock (&ioapic->lock, &fia);
   *(volatile uint32_t*)ioapic->mmio_base = reg;
   *(volatile uint32_t*)(ioapic->mmio_base + 0x10) = value;
-  spin_unlock (&ioapic->lock);
+  spin_unlock (&ioapic->lock, fia);
 }
 
 /* Find an IOAPIC corresposting to provided IRQ */
@@ -197,7 +201,9 @@ void lapic_eoi (void) { lapic_write (LAPIC_EOI, 0); }
  * us - Period length in microseconds
  */
 static uint32_t lapic_calibrate (uint32_t us) {
-  spin_lock (&lapic_calibration_lock);
+  uint64_t flc;
+
+  spin_lock (&lapic_calibration_lock, &flc);
 
   lapic_write (LAPIC_DCR, DIVIDER_VALUE);
 
@@ -210,7 +216,7 @@ static uint32_t lapic_calibrate (uint32_t us) {
   uint32_t ticks = 0xFFFFFFFF - lapic_read (LAPIC_TIMCCT);
   DEBUG ("timer ticks = %u\n", ticks);
 
-  spin_unlock (&lapic_calibration_lock);
+  spin_unlock (&lapic_calibration_lock, flc);
 
   return ticks;
 }

kernel/amd64/debug.c

@@ -33,6 +33,8 @@ static void debug_serial_write (char x) {
  * Formatted printing to serial. serial_lock ensures that all prints are atomic.
  */
 void debugprintf (const char* fmt, ...) {
+  uint64_t f1;
+
   if (!debug_is_init)
     return;
 
@@ -48,14 +50,14 @@ void debugprintf (const char* fmt, ...) {
 
   const char* p = buffer;
 
-  spin_lock (&serial_lock);
+  spin_lock (&serial_lock, &f1);
 
   while (*p) {
     debug_serial_write (*p);
     p++;
   }
 
-  spin_unlock (&serial_lock);
+  spin_unlock (&serial_lock, f1);
 }
 
 /* Initialize serial */

kernel/amd64/hpet.c

@@ -59,8 +59,10 @@ static void hpet_write32 (uint32_t reg, uint32_t value) {
 /* Read current value of HPET_MCVR register. */
 
 static uint64_t hpet_read_counter (void) {
+  uint64_t fh;
+
   uint64_t value;
-  spin_lock (&hpet_lock);
+  spin_lock (&hpet_lock, &fh);
 
   if (!hpet_32bits)
     value = hpet_read64 (HPET_MCVR);
@@ -75,13 +77,15 @@ static uint64_t hpet_read_counter (void) {
     value = ((uint64_t)hi1 << 32) | lo;
   }
 
-  spin_unlock (&hpet_lock);
+  spin_unlock (&hpet_lock, fh);
 
   return value;
 }
 
 static void hpet_write_counter (uint64_t value) {
-  spin_lock (&hpet_lock);
+  uint64_t fh;
+
+  spin_lock (&hpet_lock, &fh);
 
   if (!hpet_32bits)
     hpet_write64 (HPET_MCVR, value);
@@ -90,7 +94,7 @@ static void hpet_write_counter (uint64_t value) {
     hpet_write32 (HPET_MCVR + 4, (uint32_t)(value >> 32));
   }
 
-  spin_unlock (&hpet_lock);
+  spin_unlock (&hpet_lock, fh);
 }
 
 /* Sleep for a given amount of microseconds. This time can last longer due to \ref hpet_lock being

kernel/amd64/intr.c

@@ -191,20 +191,22 @@ void intr_handler (void* stack_ptr) {
   if (regs->trap <= 31) {
     intr_exception (regs);
   } else {
-    spin_lock (&thiscpu->lock);
+    uint64_t ftc, fpc;
+
+    spin_lock (&thiscpu->lock, &ftc);
 
     struct proc* proc_current = thiscpu->proc_current;
 
     if (proc_current != NULL) {
-      spin_lock (&proc_current->lock);
+      spin_lock (&proc_current->lock, &fpc);
       memcpy (&proc_current->pdata.regs, regs, sizeof (struct saved_regs));
 
       fx_save (proc_current->pdata.fx_env);
 
-      spin_unlock (&proc_current->lock);
+      spin_unlock (&proc_current->lock, fpc);
     }
 
-    spin_unlock (&thiscpu->lock);
+    spin_unlock (&thiscpu->lock, ftc);
 
     lapic_eoi ();
 

kernel/amd64/intr_defs.h

@@ -1,9 +1,9 @@
 #ifndef _KERNEL_AMD64_INTR_DEFS_H
 #define _KERNEL_AMD64_INTR_DEFS_H
 
-#define PS2KB 32
-#define IDE_DRIVE_PRIM      33
-#define IDE_DRIVE_SCND      34
+#define PS2KB          32
+#define IDE_DRIVE_PRIM 33
+#define IDE_DRIVE_SCND 34
 
 #define SCHED_PREEMPT_TIMER 80
 #define CPU_REQUEST_SCHED   82

kernel/amd64/mm.c

@@ -23,11 +23,6 @@ struct pg_index {
 
 /* Kernel page directory */
 static struct pd kernel_pd;
-static spin_lock_t kernel_pd_lock;
-
-void mm_kernel_lock (void) { spin_lock (&kernel_pd_lock); }
-
-void mm_kernel_unlock (void) { spin_lock (&kernel_pd_lock); }
 
 /* Get current value of CR3 register */
 static uintptr_t current_cr3 (void) {

kernel/amd64/proc.c

@@ -68,6 +68,7 @@ struct proc* proc_from_elf (uint8_t* elf_contents) {
 
 struct proc* proc_clone (struct proc* proto, uintptr_t vstack_top, uintptr_t entry,
                          uintptr_t argument_ptr) {
+  uint64_t fpt;
   struct limine_hhdm_response* hhdm = limine_hhdm_request.response;
 
   struct proc* proc = malloc (sizeof (*proc));
@@ -85,12 +86,12 @@ struct proc* proc_clone (struct proc* proto, uintptr_t vstack_top, uintptr_t ent
     return NULL;
   }
 
-  spin_lock (&proto->lock);
+  spin_lock (&proto->lock, &fpt);
 
   proc->procgroup = proto->procgroup;
   procgroup_attach (proc->procgroup, proc);
 
-  spin_unlock (&proto->lock);
+  spin_unlock (&proto->lock, fpt);
 
   uintptr_t kstack_paddr = pmm_alloc (KSTACK_SIZE / PAGE_SIZE);
   proc->pdata.kernel_stack = kstack_paddr + (uintptr_t)hhdm->offset + KSTACK_SIZE;
@@ -112,25 +113,27 @@ struct proc* proc_clone (struct proc* proto, uintptr_t vstack_top, uintptr_t ent
 }
 
 void proc_cleanup (struct proc* proc, struct reschedule_ctx* rctx) {
-  spin_lock (&proc->lock);
-  spin_lock (&proc->done_sq.lock);
+  uint64_t fp, fsq;
+
+  spin_lock (&proc->lock, &fp);
+  spin_lock (&proc->done_sq.lock, &fsq);
 
   while (proc->done_sq.proc_list != NULL) {
     struct list_node_link* node = proc->done_sq.proc_list;
     struct proc_sq_entry* sq_entry = list_entry (node, struct proc_sq_entry, sq_link);
     struct proc* suspended_proc = sq_entry->proc;
 
-    spin_unlock (&proc->done_sq.lock);
-    spin_unlock (&proc->lock);
+    spin_unlock (&proc->done_sq.lock, fsq);
+    spin_unlock (&proc->lock, fp);
 
     proc_sq_resume (suspended_proc, sq_entry, rctx);
 
-    spin_lock (&proc->lock);
-    spin_lock (&proc->done_sq.lock);
+    spin_lock (&proc->lock, &fp);
+    spin_lock (&proc->done_sq.lock, &fsq);
   }
 
-  spin_unlock (&proc->done_sq.lock);
-  spin_unlock (&proc->lock);
+  spin_unlock (&proc->done_sq.lock, fsq);
+  spin_unlock (&proc->lock, fp);
 
   proc_sqs_cleanup (proc);
   proc_mutexes_cleanup (proc, rctx);

kernel/amd64/sched1.c

@@ -11,8 +11,10 @@
 #include <sys/mm.h>
 #include <sys/smp.h>
 
-void do_sched (struct proc* proc, spin_lock_t* cpu_lock) {
-  spin_lock (&proc->lock);
+void do_sched (struct proc* proc, spin_lock_t* cpu_lock, uint64_t lockflags) {
+  uint64_t fp;
+
+  spin_lock (&proc->lock, &fp);
 
   thiscpu->tss.rsp0 = proc->pdata.kernel_stack;
   thiscpu->syscall_kernel_stack = proc->pdata.kernel_stack;
@@ -25,8 +27,8 @@ void do_sched (struct proc* proc, spin_lock_t* cpu_lock) {
 
   fx_restore (proc->pdata.fx_env);
 
-  spin_unlock (&proc->lock);
-  spin_unlock (cpu_lock);
+  spin_unlock (&proc->lock, fp);
+  spin_unlock (cpu_lock, lockflags);
 
   do_sched1 ((void*)&regs, cr3);
 }

kernel/amd64/smp.c

@@ -57,24 +57,25 @@ void cpu_request_sched (struct cpu* cpu) {
 }
 
 struct cpu* cpu_find_lightest (void) {
+  uint64_t fc;
   struct limine_mp_response* mp = limine_mp_request.response;
 
   int start = atomic_fetch_add (&last_cpu_index, 1) % mp->cpu_count;
 
   struct cpu* best_cpu = &cpus[start];
 
-  spin_lock (&best_cpu->lock);
+  spin_lock (&best_cpu->lock, &fc);
   int best_load = best_cpu->proc_run_q_count;
-  spin_unlock (&best_cpu->lock);
+  spin_unlock (&best_cpu->lock, fc);
 
   for (int i = 1; i < (int)mp->cpu_count; i++) {
     int idx = (start + i) % mp->cpu_count;
 
     struct cpu* cpu = &cpus[idx];
 
-    spin_lock (&cpu->lock);
+    spin_lock (&cpu->lock, &fc);
     int l = cpu->proc_run_q_count;
-    spin_unlock (&cpu->lock);
+    spin_unlock (&cpu->lock, fc);
 
     if (l < best_load) {
       best_load = l;
@@ -87,6 +88,8 @@ struct cpu* cpu_find_lightest (void) {
 
 /// Bootstrap code for non-BSP CPUs
 static void smp_bootstrap (struct limine_mp_info* mp_info) {
+  uint64_t fc;
+
   load_kernel_cr3 ();
 
   struct cpu* cpu = cpu_make (mp_info->lapic_id, mp_info->processor_id);
@@ -110,8 +113,8 @@ static void smp_bootstrap (struct limine_mp_info* mp_info) {
   struct proc* spin_proc = proc_from_file (thiscpu->kproc, "RD", "/spin", &rctx);
   proc_register (spin_proc, thiscpu, &rctx);
 
-  spin_lock (&spin_proc->cpu->lock);
-  do_sched (spin_proc, &spin_proc->cpu->lock);
+  spin_lock (&spin_proc->cpu->lock, &fc);
+  do_sched (spin_proc, &spin_proc->cpu->lock, fc);
 }
 
 /// Initialize SMP subsystem for AMD64. Start AP CPUs

kernel/amd64/spin_lock.c

@@ -2,3 +2,11 @@
 
 /// Relax the spinlock using AMD64 pause instruction
 void spin_lock_relax (void) { __asm__ volatile ("pause"); }
+
+void spin_lock_save_flags (uint64_t* flags) {
+  __asm__ volatile ("pushfq; cli; popq %0" : "=rm"(*flags)::"memory");
+}
+
+void spin_lock_restore_flags (uint64_t flags) {
+  __asm__ volatile ("pushq %0; popfq" ::"rm"(flags) : "memory", "cc");
+}

kernel/amd64/syscall.c

@@ -19,20 +19,22 @@
 extern void syscall_entry (void);
 
 uintptr_t syscall_dispatch (void* stack_ptr) {
+  uint64_t fp, fc;
+
   load_kernel_cr3 ();
   struct saved_regs* regs = stack_ptr;
 
-  spin_lock (&thiscpu->lock);
+  spin_lock (&thiscpu->lock, &fc);
   struct proc* caller = thiscpu->proc_current;
   int caller_pid = caller->pid;
-  spin_lock (&caller->lock);
+  spin_lock (&caller->lock, &fp);
 
   memcpy (&caller->pdata.regs, regs, sizeof (struct saved_regs));
 
   fx_save (caller->pdata.fx_env);
 
-  spin_unlock (&caller->lock);
-  spin_unlock (&thiscpu->lock);
+  spin_unlock (&caller->lock, fp);
+  spin_unlock (&thiscpu->lock, fc);
 
   int syscall_num = regs->rax;
   syscall_handler_func_t func = syscall_find_handler (syscall_num);
@@ -50,9 +52,9 @@ uintptr_t syscall_dispatch (void* stack_ptr) {
   caller = proc_find_pid (caller_pid);
 
   if (caller != NULL) {
-    spin_lock (&caller->lock);
+    spin_lock (&caller->lock, &fp);
     caller->pdata.regs.rax = r;
-    spin_unlock (&caller->lock);
+    spin_unlock (&caller->lock, fp);
   }
 
   bool do_thiscpu = false;