PCI-IDE interrupt based driver fixes, works on QEMU

aux/write_iso.sh

@@ -5,4 +5,4 @@
 
 set -x
 
-dd if=./mop3.iso of="$1" bs=4M status=progress conv=notrunc,sync && sync
+dd if=./mop3.iso of="$1" bs=4M status=progress oflag=direct conv=notrunc,sync && sync

kernel/amd64/apic.c

@@ -114,26 +114,45 @@ void ioapic_route_irq (uint32_t vec, uint32_t irq, uint64_t flags, uint64_t lapi
     }
   }
 
-  uint64_t calc_flags = (lapic_id << 56) | (flags) | (vec & 0xFF);
+  uint64_t gsi = found_override ? override->gsi : irq;
+
+  uint64_t redirflags = 0;
+  redirflags |= (vec & 0xFF);
+  redirflags |= flags;
+  redirflags |= (lapic_id << 56);
 
   if (found_override) {
-    uint32_t polarity = ((override->flags & 0x03) == 0x03) ? 1 : 0;
+    switch (override->flags & 0x03) {
-    uint32_t mode = (((override->flags >> 2) & 0x03) == 0x03) ? 1 : 0;
+    case 0:
-    calc_flags |= (uint64_t)mode << 15;
+      break;
-    calc_flags |= (uint64_t)polarity << 13;
+    case 1:
-  }
+      redirflags &= ~(1 << 13);
+      break;
+    case 3:
+      redirflags |= (1 << 13);
+      break;
+    }
 
-  uint32_t gsi = found_override ? override->gsi : irq;
+    switch ((override->flags >> 2) & 0x03) {
+    case 0:
+      break;
+    case 1:
+      redirflags &= ~(1 << 15);
+      break;
+    case 3:
+      redirflags |= (1 << 15);
+      break;
+    }
+  }
 
   ioapic = ioapic_find (gsi);
 
-  if (ioapic == NULL)
+  if (!ioapic)
     return;
 
   uint32_t irq_reg = ((gsi - ioapic->table_data.gsi_base) * 2) + 0x10;
-
+  ioapic_write (ioapic, irq_reg + 1, (uint32_t)(redirflags >> 32));
-  ioapic_write (ioapic, irq_reg + 1, (uint32_t)(calc_flags >> 32));
+  ioapic_write (ioapic, irq_reg, (uint32_t)redirflags);
-  ioapic_write (ioapic, irq_reg, (uint32_t)calc_flags);
 }
 
 /* Find and initialize the IOAPIC */
@@ -171,6 +190,9 @@ void ioapic_init (void) {
       struct acpi_madt_interrupt_source_override* override =
           (struct acpi_madt_interrupt_source_override*)current;
       intr_src_overrides[intr_src_override_entries++] = *override;
+
+      DEBUG ("Interrupt override: source=%u gsi=%u flags=%04x bus=%u\n", override->source,
+             override->gsi, override->flags, override->bus);
     } break;
     }
 

kernel/device/idedrv.c

@@ -41,13 +41,12 @@
 
 #define IDE_READ  1
 #define IDE_WRITE 2
-#define IDE_FLUSH 3
 
 static bool ide_wait (uint16_t io, uint32_t timeout, bool drq, bool errcheck) {
   uint32_t i = 0;
   uint8_t status;
 
-  for (;;) {
+  while (1) {
     status = inb (io + IDE_REG_STATUS);
 
     if (!(status & IDE_BSY))
@@ -57,26 +56,26 @@ static bool ide_wait (uint16_t io, uint32_t timeout, bool drq, bool errcheck) {
       return false;
   }
 
-  if (drq) {
+  if (errcheck && (status & (IDE_ERR | IDE_DF)))
-    i = 0;
-    while (!(status & IDE_DRQ) || (status & IDE_BSY)) {
-      if (status & IDE_ERR)
-        return false;
-
-      status = inb (io + IDE_REG_STATUS);
-
-      if (status == 0xFF)
-        return false;
-
-      if (++i >= timeout)
-        return false;
-    }
-  }
-
-  if (errcheck && (status & (IDE_DF | IDE_ERR)))
     return false;
 
-  return true;
+  if (!drq)
+    return true;
+
+  i = 0;
+
+  while (1) {
+    status = inb (io + IDE_REG_STATUS);
+
+    if (status & (IDE_ERR | IDE_DF))
+      return false;
+
+    if (status & IDE_DRQ)
+      return true;
+
+    if (++i >= timeout)
+      return false;
+  }
 }
 
 #pragma clang optimize off
@@ -92,60 +91,50 @@ static void ide_irq (void* arg, void* regs, bool user, struct reschedule_ctx* rc
   uint64_t fd, fp;
 
   struct idedrv* idedrv = arg;
+  DEBUG ("1\n");
 
   spin_lock (&idedrv->device->lock, &fd);
 
-  struct list_node_link* node = idedrv->requests;
+  struct idedrv_request* req = idedrv->current_req;
+  DEBUG ("2\n");
 
-  if (node == NULL) {
+  if (req == NULL) {
     (void)inb (idedrv->io + IDE_REG_STATUS);
     spin_unlock (&idedrv->device->lock, fd);
     return;
   }
-
+  DEBUG ("3\n");
-  struct idedrv_request* req = list_entry (node, struct idedrv_request, requests_link);
 
   uint8_t status = inb (idedrv->io + IDE_REG_STATUS);
 
-  if (req->type == IDE_FLUSH) {
-    list_remove (idedrv->requests, &req->requests_link);
-    spin_unlock (&idedrv->device->lock, fd);
-    atomic_store (&req->done, 1);
-    return;
-  }
-
   if ((status & (IDE_ERR | IDE_DF))) {
-    list_remove (idedrv->requests, &req->requests_link);
-    spin_unlock (&idedrv->device->lock, fd);
     atomic_store (&req->done, 1);
+    idedrv->current_req = NULL;
+    spin_unlock (&idedrv->device->lock, fd);
     return;
   }
+  DEBUG ("4\n");
 
-  if (!(status & IDE_BSY) && (status & IDE_DRQ)) {
+  if ((status & IDE_DRQ) && (req->sector_done_count < req->sector_count)) {
-    switch (req->type) {
+    uint16_t* p = req->buffer + (req->sector_done_count * (idedrv->sector_size / 2));
-    case IDE_READ: {
+
-      uint16_t* p = req->buffer + (req->sector_done_count * (idedrv->sector_size / 2));
+    if (req->type == IDE_READ)
       insw (idedrv->io + IDE_REG_DATA, p, idedrv->sector_size / 2);
-      req->sector_done_count++;
+    else
-    } break;
+      outsw (idedrv->io + IDE_REG_DATA, p, idedrv->sector_size / 2);
-    case IDE_WRITE: {
+
-      req->sector_done_count++;
+    req->sector_done_count++;
-      if (req->sector_done_count < req->sector_count) {
+    DEBUG ("4\n");
-        uint16_t* p = req->buffer + (req->sector_done_count * (idedrv->sector_size / 2));
-        outsw (idedrv->io + IDE_REG_DATA, p, idedrv->sector_size / 2);
-      }
-    } break;
-    }
   }
 
-  if (req->sector_done_count >= req->sector_count) {
+  if ((req->sector_done_count >= req->sector_count)) {
-    list_remove (idedrv->requests, &req->requests_link);
-    spin_unlock (&idedrv->device->lock, fd);
     atomic_store (&req->done, 1);
-    return;
+    idedrv->current_req = NULL;
   }
+  DEBUG ("6\n");
 
   spin_unlock (&idedrv->device->lock, fd);
+  DEBUG ("7\n");
 }
 
 void ide_probe (uint16_t io, uint16_t ctrl, uint8_t devno, struct ide_probe* probe) {
@@ -211,14 +200,21 @@ void ide_probe (uint16_t io, uint16_t ctrl, uint8_t devno, struct ide_probe* pro
     probe->sector_size = 512;
 }
 
-static void ide_prepare (struct idedrv* idedrv, size_t sector, uint16_t sector_count, bool clear) {
+static void ide_prepare (struct idedrv* idedrv, size_t sector, uint16_t sector_count,
-  if (clear)
+                         bool irq_enable) {
-    outb (idedrv->ctrl, 0x00);
+  uint8_t ctrl = inb (idedrv->ctrl);
+  if (irq_enable)
+    ctrl &= ~0x02;
+  else
+    ctrl |= 0x02;
+  outb (idedrv->ctrl, ctrl);
 
   if (idedrv->lba48) {
     outb (idedrv->io + IDE_REG_DRIVE, 0x40 | (idedrv->devno << 4));
     ide_delay (idedrv->ctrl);
 
+    ide_wait (idedrv->io, 100000, false, false);
+
     outb (idedrv->io + IDE_REG_SECCOUNT, (sector_count >> 8) & 0xFF);
     outb (idedrv->io + IDE_REG_LBA0, (sector >> 24) & 0xFF);
     outb (idedrv->io + IDE_REG_LBA1, (sector >> 32) & 0xFF);
@@ -229,9 +225,11 @@ static void ide_prepare (struct idedrv* idedrv, size_t sector, uint16_t sector_c
     outb (idedrv->io + IDE_REG_LBA1, (sector >> 8) & 0xFF);
     outb (idedrv->io + IDE_REG_LBA2, (sector >> 16) & 0xFF);
   } else {
-    outb (idedrv->io + IDE_REG_DRIVE, 0xE0 | (idedrv->devno << 4) | ((sector >> 24) & 0xFF));
+    outb (idedrv->io + IDE_REG_DRIVE, 0xE0 | (idedrv->devno << 4) | ((sector >> 24) & 0x0F));
     ide_delay (idedrv->ctrl);
 
+    ide_wait (idedrv->io, 100000, false, false);
+
     uint8_t count = (sector_count == 256) ? 0 : (uint8_t)sector_count;
     outb (idedrv->io + IDE_REG_SECCOUNT, count);
     outb (idedrv->io + IDE_REG_LBA0, sector & 0xFF);
@@ -258,17 +256,12 @@ bool idedrv_init (struct device* device, void* arg, struct proc* proc,
   idedrv->io = init->io;
   idedrv->ctrl = init->ctrl;
   idedrv->devno = init->devno;
-  idedrv->primscnd = init->primscnd;
+  idedrv->irq = init->irq;
+  idedrv->current_req = NULL;
 
   device->udata = idedrv;
 
-  if (idedrv->primscnd == 1) {
+  irq_attach (&ide_irq, idedrv, idedrv->irq);
-    ioapic_route_irq (IDE_DRIVE_PRIM, 14, 0, thiscpu->lapic_id);
-    irq_attach (&ide_irq, idedrv, IDE_DRIVE_PRIM);
-  } else if (idedrv->primscnd == 2) {
-    ioapic_route_irq (IDE_DRIVE_SCND, 15, 0, thiscpu->lapic_id);
-    irq_attach (&ide_irq, idedrv, IDE_DRIVE_SCND);
-  }
 
   return true;
 }
@@ -276,14 +269,12 @@ bool idedrv_init (struct device* device, void* arg, struct proc* proc,
 void idedrv_fini (struct device* device, struct proc* proc, struct reschedule_ctx* rctx) {
   struct idedrv* idedrv = device->udata;
 
-  struct list_node_link *req_link, *tmp_req_link;
+  if (idedrv->current_req != NULL) {
-  list_foreach (idedrv->requests, req_link, tmp_req_link) {
+    free (idedrv->current_req);
-    struct idedrv_request* req = list_entry (req_link, struct idedrv_request, requests_link);
+    idedrv->current_req = NULL;
-    list_remove (idedrv->requests, &req->requests_link);
-    free (req);
   }
 
-  irq_detach (idedrv->primscnd == 1 ? IDE_DRIVE_PRIM : IDE_DRIVE_SCND);
+  irq_detach (idedrv->irq);
   free (idedrv);
 }
 
@@ -300,9 +291,14 @@ int idedrv_read (struct device* device, struct proc* proc, struct reschedule_ctx
 
   struct idedrv* idedrv = device->udata;
 
+  DEBUG ("1\n");
+
   if (sector + sector_count > idedrv->sector_count)
     return -ST_OOB_ERROR;
 
+  if (!ide_wait (idedrv->io, 100000, false, false))
+    return -ST_XDRV_READ_ERROR;
+
   struct idedrv_request* req = malloc (sizeof (*req));
 
   if (req == NULL)
@@ -313,23 +309,18 @@ int idedrv_read (struct device* device, struct proc* proc, struct reschedule_ctx
   req->sector_count = sector_count;
   req->sector_done_count = 0;
   req->type = IDE_READ;
+  DEBUG ("2\n");
 
-  list_append (idedrv->requests, &req->requests_link);
+  idedrv->current_req = req;
 
   ide_prepare (idedrv, sector, sector_count, true);
+  DEBUG ("3\n");
+
+  DEBUG ("4\n");
 
   uint8_t cmd = idedrv->lba48 ? IDE_CMD_READ48 : IDE_CMD_READ28;
   outb (idedrv->io + IDE_REG_CMD, cmd);
-
+  DEBUG ("5\n");
-  if (!ide_wait (idedrv->io, 100000, true, true)) {
-    list_remove (idedrv->requests, &req->requests_link);
-    free (req);
-    return -ST_XDRV_READ_ERROR;
-  }
-
-  insw (idedrv->io + IDE_REG_DATA, buffer, idedrv->sector_size / 2);
-
-  req->sector_done_count = 1;
 
   spin_unlock (&device->lock, *lockflags);
 
@@ -337,6 +328,7 @@ int idedrv_read (struct device* device, struct proc* proc, struct reschedule_ctx
     spin_lock_relax ();
 
   spin_lock (&device->lock, lockflags);
+  DEBUG ("6\n");
 
   free (req);
 
@@ -360,33 +352,21 @@ int idedrv_write (struct device* device, struct proc* proc, struct reschedule_ct
   if (sector + sector_count > idedrv->sector_count)
     return -ST_OOB_ERROR;
 
-  struct idedrv_request* req = malloc (sizeof (*req));
+  if (!ide_wait (idedrv->io, 100000, false, false))
+    return -ST_XDRV_WRITE_ERROR;
 
-  struct idedrv_request* flushreq = malloc (sizeof (*req));
+  struct idedrv_request* req = malloc (sizeof (*req));
 
   if (req == NULL)
     return -ST_OOM_ERROR;
 
-  if (flushreq == NULL) {
-    free (req);
-    return -ST_OOM_ERROR;
-  }
-
   memset (req, 0, sizeof (*req));
   req->buffer = buffer;
   req->sector_count = sector_count;
   req->sector_done_count = 0;
   req->type = IDE_WRITE;
 
-  list_append (idedrv->requests, &req->requests_link);
+  idedrv->current_req = req;
-
-  memset (flushreq, 0, sizeof (*flushreq));
-  flushreq->buffer = NULL;
-  flushreq->sector_count = 0;
-  flushreq->sector_done_count = 0;
-  flushreq->type = IDE_FLUSH;
-
-  list_append (idedrv->requests, &flushreq->requests_link);
 
   ide_prepare (idedrv, sector, sector_count, true);
 
@@ -394,7 +374,7 @@ int idedrv_write (struct device* device, struct proc* proc, struct reschedule_ct
   outb (idedrv->io + IDE_REG_CMD, cmd);
 
   if (!ide_wait (idedrv->io, 100000, true, true)) {
-    list_remove (idedrv->requests, &req->requests_link);
+    idedrv->current_req = NULL;
     free (req);
     return -ST_XDRV_WRITE_ERROR;
   }
@@ -410,20 +390,24 @@ int idedrv_write (struct device* device, struct proc* proc, struct reschedule_ct
 
   spin_lock (&device->lock, lockflags);
 
+  free (req);
+
+  uint8_t ctrl = inb (idedrv->ctrl);
+  ctrl |= 0x02;
+  outb (idedrv->ctrl, ctrl);
+
   if (idedrv->lba48)
     outb (idedrv->io + IDE_REG_CMD, IDE_CMD_FLUSH48);
   else
     outb (idedrv->io + IDE_REG_CMD, IDE_CMD_FLUSH28);
 
-  spin_unlock (&device->lock, *lockflags);
+  uint8_t status;
+  do {
+    status = inb (idedrv->io + IDE_REG_STATUS);
+  } while (status & IDE_BSY);
 
-  while (!atomic_load (&flushreq->done))
+  if (status & (IDE_ERR | IDE_DF))
-    spin_lock_relax ();
+    return -ST_XDRV_WRITE_ERROR;
-
-  spin_lock (&device->lock, lockflags);
-
-  free (req);
-  free (flushreq);
 
   return ST_OK;
 }

kernel/device/idedrv.h

@@ -19,14 +19,13 @@ struct idedrv_init {
   size_t sector_size;
   uint16_t io, ctrl;
   uint8_t devno;
-  uint8_t primscnd;
+  uint8_t irq;
 };
 
 struct idedrv_request {
   uint16_t* buffer;
   size_t sector_done_count;
   size_t sector_count;
-  struct list_node_link requests_link;
   int type;
   atomic_int done;
 };
@@ -38,9 +37,8 @@ struct idedrv {
   size_t sector_size;
   uint16_t io, ctrl;
   uint8_t devno;
-  uint8_t primscnd;
+  uint8_t irq;
-
+  struct idedrv_request* current_req;
-  struct list_node_link* requests;
 };
 
 struct ide_probe {
@@ -49,7 +47,7 @@ struct ide_probe {
   size_t sector_size;
   uint16_t io, ctrl;
   uint8_t devno;
-  uint8_t primscnd;
+  uint8_t irq;
 };
 
 bool idedrv_init (struct device* device, void* arg, struct proc* proc, struct reschedule_ctx* rctx);

kernel/device/partitions.c

@@ -30,11 +30,14 @@ static int device_probe_partitions_dos (struct proc* proc, struct reschedule_ctx
       [XDRV_WRITE] = &partdrv_write,
   };
 
+  DEBUG ("1\n");
   spin_lock (&device->lock, &fd);
 
   device_op (device, XDRV_GET_SECTOR_SIZE, proc, rctx, &fd, &sector_size);
 
+  DEBUG ("2\n");
   int ret = device_op (device, XDRV_READ, proc, rctx, &fd, &sector, &sector_count, &mbr);
+  DEBUG ("3\n");
 
   if (ret < 0) {
     spin_unlock (&device->lock, fd);
@@ -45,6 +48,7 @@ static int device_probe_partitions_dos (struct proc* proc, struct reschedule_ctx
     spin_unlock (&device->lock, fd);
     return -ST_PARTITION_ERROR;
   }
+  DEBUG ("4\n");
 
   for (size_t i = 0; i < lengthof (mbr.ptes); i++) {
     struct dos_pte* pte = &mbr.ptes[i];
@@ -60,9 +64,11 @@ static int device_probe_partitions_dos (struct proc* proc, struct reschedule_ctx
     snprintf (key, sizeof (key), "%sp%zu", device->key, i);
 
     device_create (key, ops, lengthof (ops), &partdrv_init, &partdrv_fini, &init, proc, rctx);
+    DEBUG ("5\n");
   }
 
   spin_unlock (&device->lock, fd);
+  DEBUG ("6\n");
 
   return ST_OK;
 }

kernel/device/pci_ide.c

@@ -1,3 +1,5 @@
+#include <amd64/apic.h>
+#include <amd64/intr_defs.h>
 #include <amd64/io.h>
 #include <device/device.h>
 #include <device/idedrv.h>
@@ -53,7 +55,7 @@ static void ide_make_device (struct proc* proc, struct reschedule_ctx* rctx,
       .io = probe.io,
       .ctrl = probe.ctrl,
       .devno = probe.devno,
-      .primscnd = probe.primscnd,
+      .irq = probe.irq,
   };
   struct device* ide = device_create (device_key, ops, lengthof (ops), &idedrv_init, &idedrv_fini,
                                       &init, proc, rctx);
@@ -63,7 +65,7 @@ static void ide_make_device (struct proc* proc, struct reschedule_ctx* rctx,
 bool pci_ide_init (struct proc* proc, struct reschedule_ctx* rctx, struct pci_info pci_info) {
   uint16_t pci_cmd = pci_read16 (pci_info.bus, pci_info.slot, pci_info.func, PCI_COMMAND);
 
-  uint16_t new_cmd = (pci_cmd | (1 << 0)) & ~(1 << 10);
+  uint16_t new_cmd = (pci_cmd | (1 << 0) | (1 << 2)) & ~(1 << 10);
 
   if (pci_cmd != new_cmd) {
     pci_write16 (pci_info.bus, pci_info.slot, pci_info.func, PCI_COMMAND, new_cmd);
@@ -88,11 +90,21 @@ bool pci_ide_init (struct proc* proc, struct reschedule_ctx* rctx, struct pci_in
     pcmd = (uint16_t)(bar0 & 0xFFFC);
     pctrl = (uint16_t)(bar1 & 0xFFFC);
 
-    if (pctrl)
+    uint8_t irq = pci_read8 (pci_info.bus, pci_info.slot, pci_info.func, PCI_INTERRUPT);
-      pctrl += 2;
+
+    if (irq == 0xFF) {
+      pci_write8 (pci_info.bus, pci_info.slot, pci_info.func, PCI_INTERRUPT, 14);
+      irq = pci_read8 (pci_info.bus, pci_info.slot, pci_info.func, PCI_INTERRUPT);
+    }
+
+    ioapic_route_irq (IDE_DRIVE_PRIM, irq, 0, thiscpu->lapic_id);
+    DEBUG ("N pcmd=%x, pctrl=%x, irq=%u\n", pcmd, pctrl, irq);
   } else {
     pcmd = 0x1F0;
     pctrl = 0x3F6;
+
+    ioapic_route_irq (IDE_DRIVE_PRIM, 14, 0, thiscpu->lapic_id);
+    DEBUG ("L pcmd=%x, pctrl=%x, irq=%u\n", pcmd, pctrl, 14);
   }
 
   if ((progif & 0x04)) {
@@ -107,27 +119,40 @@ bool pci_ide_init (struct proc* proc, struct reschedule_ctx* rctx, struct pci_in
     scmd = (uint16_t)(bar2 & 0xFFFC);
     sctrl = (uint16_t)(bar3 & 0xFFFC);
 
-    if (sctrl)
+    uint8_t irq = pci_read8 (pci_info.bus, pci_info.slot, pci_info.func, PCI_INTERRUPT);
-      sctrl += 2;
+
+    if (irq == 0xFF) {
+      pci_write8 (pci_info.bus, pci_info.slot, pci_info.func, PCI_INTERRUPT, 15);
+      irq = pci_read8 (pci_info.bus, pci_info.slot, pci_info.func, PCI_INTERRUPT);
+    }
+
+    ioapic_route_irq (IDE_DRIVE_SCND, irq, 0, thiscpu->lapic_id);
+    DEBUG ("N pcmd=%x, pctrl=%x, irq=%u\n", scmd, sctrl, irq);
   } else {
     scmd = 0x170;
     sctrl = 0x376;
+
+    ioapic_route_irq (IDE_DRIVE_SCND, 15, 0, thiscpu->lapic_id);
+    DEBUG ("L pcmd=%x, pctrl=%x, irq=%u\n", scmd, sctrl, 15);
   }
 
-  uint16_t channels[2][2] = {{pcmd, pctrl}, {scmd, sctrl}};
+  uint16_t channels[2][3] = {{pcmd, pctrl, IDE_DRIVE_PRIM}, {scmd, sctrl, IDE_DRIVE_SCND}};
 
   for (size_t i = 0; i < lengthof (channels); i++) {
     uint16_t cmd = channels[i][0];
     uint16_t ctrl = channels[i][1];
+    uint8_t irq = channels[i][2];
 
     if (cmd == 0)
       continue;
 
-    probe.primscnd = i + 1;
-
     for (size_t dev = 0; dev < 2; dev++) {
       ide_probe (cmd, ctrl, dev, &probe);
 
+      probe.ctrl = ctrl;
+      probe.io = cmd;
+      probe.irq = irq;
+
       if ((probe.flags & IDE_PROBE_AVAIL))
         ide_make_device (proc, rctx, probe);
     }