Nonblocking mail_receive, fix proc_kill scheduling issues

ce/interp.c

@@ -344,7 +344,7 @@ static void cmd_cancel_proc (void* arg) {
 
   char ch = 0;
   do {
-    mail_receive (&ch, 1);
+    mail_receive_nonblock (&ch, 1);
   } while (ch != KB_CTRL ('C'));
 
   kill (pid);
@@ -356,10 +356,10 @@ static void cmd_collect_proc (void* arg) {
   for (;;) {
     char recv[RECV_MAX];
     memset (recv, 0, sizeof (recv));
-    mail_receive (&recv, sizeof (recv) - 1);
-
+    if (mail_receive_nonblock (&recv, sizeof (recv) - 1) == ST_OK) {
       mail_send (e_pgid, recv, strlen (recv));
     }
+  }
 }
 
 static void execute_cmd (struct ast_cmd* cmd, struct context* context) {

include/syscall_defs.h

@@ -34,5 +34,6 @@
 #define SYS_ENV_GET       31
 #define SYS_EXEC_PARTIAL  32
 #define SYS_EXEC_PARTIAL_FINI 33
+#define SYS_MAIL_RECEIVE_NONBLOCK 34
 
 #endif // _M_SYSCALL_DEFS_H

kernel/amd64/intr.c

@@ -168,13 +168,13 @@ static void intr_exception (struct saved_regs* regs) {
     bool do_thiscpu = false;
     for (size_t i = 0; i < lengthof (rctx.cpus); i++) {
       if (rctx.cpus[i] != NULL && rctx.cpus[i] != thiscpu)
-        cpu_request_sched (rctx.cpus[i]);
+        cpu_request_sched (rctx.cpus[i], true);
       else
         do_thiscpu = true;
     }
 
     if (do_thiscpu)
-      cpu_request_sched (thiscpu);
+      cpu_request_sched (thiscpu, true);
   } else {
     spin ();
   }
@@ -228,13 +228,13 @@ void intr_handler (void* stack_ptr) {
       bool do_thiscpu = false;
       for (size_t i = 0; i < lengthof (rctx.cpus); i++) {
         if (rctx.cpus[i] != NULL && rctx.cpus[i] != thiscpu)
-          cpu_request_sched (rctx.cpus[i]);
+          cpu_request_sched (rctx.cpus[i], user);
         else
           do_thiscpu = true;
       }
 
       if (do_thiscpu)
-        cpu_request_sched (thiscpu);
+        cpu_request_sched (thiscpu, user);
     }
   }
 }

kernel/amd64/smp.c

@@ -46,9 +46,9 @@ struct cpu* cpu_get (void) {
   return ptr;
 }
 
-void cpu_request_sched (struct cpu* cpu) {
+void cpu_request_sched (struct cpu* cpu, bool user) {
   if (cpu == thiscpu) {
-    proc_sched ();
+    proc_sched (user);
     return;
   }
 

kernel/amd64/smp.h

@@ -37,8 +37,11 @@ struct cpu {
 };
 
 struct cpu* cpu_make (uint64_t lapic_id, uint64_t acpi_id);
+
 struct cpu* cpu_get (void);
-void cpu_request_sched (struct cpu* cpu);
+
+void cpu_request_sched (struct cpu* cpu, bool user);
+
 struct cpu* cpu_find_lightest (void);
 
 #define thiscpu (cpu_get ())

kernel/amd64/syscall.c

@@ -69,13 +69,13 @@ uintptr_t syscall_dispatch (void* stack_ptr) {
   bool do_thiscpu = false;
   for (size_t i = 0; i < lengthof (rctx.cpus); i++) {
     if (rctx.cpus[i] != NULL && rctx.cpus[i] != thiscpu)
-      cpu_request_sched (rctx.cpus[i]);
+      cpu_request_sched (rctx.cpus[i], true);
     else
       do_thiscpu = true;
   }
 
   if (do_thiscpu)
-    cpu_request_sched (thiscpu);
+    cpu_request_sched (thiscpu, true);
 
   return r;
 }

kernel/proc/mail.c

@@ -147,3 +147,51 @@ void proc_mail_receive (struct proc* proc, struct proc_mail* mail, struct resche
   /* nothing to receive */
   proc_sq_suspend (proc, &mail->recv_sq, &mail->resource->lock, fr, rctx);
 }
+
+bool proc_mail_receive_nonblock (struct proc* proc, struct proc_mail* mail,
+                                 struct reschedule_ctx* rctx, void* recv_buffer, size_t recv_size) {
+  uint64_t fp, fr, fssq;
+
+  spin_lock (&proc->lock, &fp);
+  proc->mail_recv_buffer = recv_buffer;
+  proc->mail_recv_size = recv_size;
+  spin_unlock (&proc->lock, fp);
+
+  spin_lock (&mail->resource->lock, &fr);
+
+  /* consume mesg if available */
+  if (mail->packets_count > 0) {
+    struct mail_packet* packet = list_entry (mail->packets, struct mail_packet, packets_link);
+    list_remove (mail->packets, &packet->packets_link);
+    mail->packets_count--;
+
+    memcpy (recv_buffer, packet->packet_buffer, min (recv_size, packet->packet_size));
+    free (packet->packet_buffer);
+    free (packet);
+
+    /* check for suspended sender */
+    spin_lock (&mail->send_sq.lock, &fssq);
+
+    struct list_node_link* node = mail->send_sq.proc_list;
+
+    if (node != NULL) {
+      struct proc_sq_entry* sq_entry = list_entry (node, struct proc_sq_entry, sq_link);
+      struct proc* resumed_proc = sq_entry->proc;
+
+      spin_unlock (&mail->send_sq.lock, fssq);
+      spin_unlock (&mail->resource->lock, fr);
+
+      proc_sq_resume (resumed_proc, sq_entry, rctx);
+      return true;
+    }
+
+    spin_unlock (&mail->send_sq.lock, fssq);
+    spin_unlock (&mail->resource->lock, fr);
+
+    return true;
+  }
+
+  spin_unlock (&mail->resource->lock, fr);
+
+  return false;
+}

kernel/proc/mail.h

@@ -37,4 +37,7 @@ void proc_mail_send (struct proc* proc, struct proc_mail* mail, struct reschedul
 void proc_mail_receive (struct proc* proc, struct proc_mail* mail, struct reschedule_ctx* rctx,
                         void* recv_buffer, size_t recv_size);
 
+bool proc_mail_receive_nonblock (struct proc* proc, struct proc_mail* mail,
+                                 struct reschedule_ctx* rctx, void* recv_buffer, size_t recv_size);
+
 #endif // _KERNEL_PROC_MAIL_H

kernel/proc/proc.c

@@ -273,40 +273,7 @@ static struct proc* proc_find_sched (struct cpu* cpu) {
   return NULL;
 }
 
-static void proc_reaper (struct reschedule_ctx* rctx) {
-  uint64_t fpt, fp, fc;
-
-  struct list_node_link* reaper_list = NULL;
-
-  spin_lock (&proc_tree_lock, &fpt);
-  spin_lock (&thiscpu->lock, &fc);
-
-  struct list_node_link *run_link, *tmp_run_link;
-  list_foreach (thiscpu->proc_run_q, run_link, tmp_run_link) {
-    struct proc* proc = list_entry (run_link, struct proc, cpu_run_q_link);
-
-    if (!proc->dead)
-      continue;
-
-    spin_lock (&proc->lock, &fp);
-    list_remove (thiscpu->proc_run_q, &proc->cpu_run_q_link);
-    rbtree_delete (&proc_tree, &proc->proc_tree_link);
-    list_append (reaper_list, &proc->reaper_list_link);
-    spin_unlock (&proc->lock, fp);
-  }
-
-  spin_unlock (&thiscpu->lock, fc);
-  spin_unlock (&proc_tree_lock, fpt);
-
-  struct list_node_link *rlink, *tmp_rlink;
-  list_foreach (reaper_list, rlink, tmp_rlink) {
-    struct proc* proc = list_entry (rlink, struct proc, reaper_list_link);
-    list_remove (reaper_list, &proc->reaper_list_link);
-    proc_cleanup (proc, rctx);
-  }
-}
-
-void proc_sched (void) {
+void proc_sched (bool user) {
   struct proc* next = NULL;
   struct cpu* cpu = thiscpu;
   uint64_t fc;
@@ -319,7 +286,10 @@ retry:
   if (next) {
     cpu->proc_current = next;
 
+    if (user)
       do_sched (next, &cpu->lock, fc);
+    else
+      spin_unlock (&cpu->lock, fc);
   } else {
     cpu->proc_current = NULL;
     spin_unlock (&cpu->lock, fc);
@@ -343,22 +313,25 @@ void proc_kill (struct proc* proc, struct reschedule_ctx* rctx) {
   struct cpu* cpu = proc->cpu;
   spin_unlock (&proc->lock, fp);
 
+  spin_lock (&proc_tree_lock, &fpt);
   spin_lock (&cpu->lock, &fc);
   spin_lock (&proc->lock, &fp);
 
-  proc->cpu = NULL;
-  proc->dead = true;
-
+  list_remove (cpu->proc_run_q, &proc->cpu_run_q_link);
   cpu->proc_run_q_count--;
   if (cpu->proc_current == proc)
     cpu->proc_current = NULL;
 
+  proc->dead = true;
+  rbtree_delete (&proc_tree, &proc->proc_tree_link);
+
   spin_unlock (&proc->lock, fp);
   spin_unlock (&cpu->lock, fc);
-
-  rctx_insert_cpu (rctx, cpu);
+  spin_unlock (&proc_tree_lock, fpt);
 
   DEBUG ("killed PID %d\n", proc->pid);
+  proc_cleanup (proc, rctx);
+  rctx_insert_cpu (rctx, cpu);
 }
 
 void proc_wait_for (struct proc* proc, struct reschedule_ctx* rctx, struct proc* wait_proc) {
@@ -368,12 +341,6 @@ void proc_wait_for (struct proc* proc, struct reschedule_ctx* rctx, struct proc*
 static void proc_irq_sched (void* arg, void* regs, bool user, struct reschedule_ctx* rctx) {
   (void)arg, (void)regs, (void)rctx;
 
-  proc_reaper (rctx);
-
-  if (!user) {
-    return;
-  }
-
   rctx_insert_cpu (rctx, thiscpu);
 }
 

kernel/proc/proc.h

@@ -54,7 +54,7 @@ struct proc {
   struct proc_env env;
 };
 
-void proc_sched (void);
+void proc_sched (bool user);
 
 void proc_kill (struct proc* proc, struct reschedule_ctx* rctx);
 

kernel/proc/suspension_q.c

@@ -81,6 +81,8 @@ int proc_sq_resume (struct proc* proc, struct proc_sq_entry* sq_entry,
     proc->state = PROC_READY;
 
   list_append (cpu->proc_run_q, &proc->cpu_run_q_link);
+  if (cpu->proc_current == NULL)
+    cpu->proc_current = proc;
   cpu->proc_run_q_count++;
 
   int state = proc->state;

kernel/syscall/syscall.c

@@ -274,6 +274,35 @@ DEFINE_SYSCALL (sys_mail_receive) {
   return SYSRESULT (ST_OK);
 }
 
+/* int mail_receive_nonblock (void* recv_mesg, size_t mesg_size) */
+DEFINE_SYSCALL (sys_mail_receive_nonblock) {
+  uint64_t fp;
+
+  uintptr_t uvaddr_mesg = a1;
+  size_t mesg_size = (size_t)a2;
+
+  spin_lock (&proc->lock, &fp);
+  struct procgroup* procgroup = proc->procgroup;
+  spin_unlock (&proc->lock, fp);
+
+  void* mesg = sys_get_user_buffer (procgroup, uvaddr_mesg, mesg_size);
+
+  if (mesg == NULL)
+    return SYSRESULT (-ST_BAD_ADDRESS_SPACE);
+
+  struct proc_resource* mail_resource = proc_find_resource (procgroup, 0);
+
+  if (mail_resource == NULL)
+    return SYSRESULT (-ST_NOT_FOUND);
+
+  bool r = proc_mail_receive_nonblock (proc, &mail_resource->u.mail, rctx, mesg, mesg_size);
+
+  if (r)
+    return SYSRESULT (ST_OK);
+  else
+    return SYSRESULT (-ST_TRY_AGAIN);
+}
+
 /* int device_do (int device_id, int cmd, void* a1, void* a2, void* a3, void* a4) */
 DEFINE_SYSCALL (sys_device_do) {
   uint64_t fpg, fd, fp;
@@ -1025,6 +1054,7 @@ static syscall_handler_func_t handler_table[] = {
     [SYS_ENV_GET] = &sys_env_get,
     [SYS_EXEC_PARTIAL] = &sys_exec_partial,
     [SYS_EXEC_PARTIAL_FINI] = &sys_exec_partial_fini,
+    [SYS_MAIL_RECEIVE_NONBLOCK] = &sys_mail_receive_nonblock,
 };
 
 syscall_handler_func_t syscall_find_handler (int syscall_num) {

libsystem/system.c

@@ -61,6 +61,10 @@ int mail_receive (void* mesg, size_t mesg_size) {
   return (int)do_syscall (SYS_MAIL_RECEIVE, mesg, mesg_size);
 }
 
+int mail_receive_nonblock (void* mesg, size_t mesg_size) {
+  return (int)do_syscall (SYS_MAIL_RECEIVE_NONBLOCK, mesg, mesg_size);
+}
+
 int get_procgroup (int pid) { return (int)do_syscall (SYS_GET_PROCGROUP, pid); }
 
 int get_exec_pid (void) { return (int)do_syscall (SYS_GET_EXEC_PID, 0); }

libsystem/system.h

@@ -63,6 +63,9 @@ int mail_send (int pgid, void* mesg, size_t mesg_size);
 /* receive a message from mail */
 int mail_receive (void* mesg, size_t mesg_size);
 
+/* receive a message from mail without blocking */
+int mail_receive_nonblock (void* mesg, size_t mesg_size);
+
 /* get procgroup id of a perticular process */
 int get_procgroup (int pid);