PCI-IDE driver fallback to polling for PCI-native controllers

aux/write_iso.sh

@@ -5,4 +5,4 @@
 
 set -x
 
-dd if=./mop3.iso of="$1" bs=4M status=progress conv=notrunc,sync && sync
+dd if=./mop3.iso of="$1" bs=4M status=progress oflag=direct conv=notrunc,sync && sync

kernel/amd64/apic.c

@@ -114,26 +114,45 @@ void ioapic_route_irq (uint32_t vec, uint32_t irq, uint64_t flags, uint64_t lapi
     }
   }
 
-  uint64_t calc_flags = (lapic_id << 56) | (flags) | (vec & 0xFF);
+  uint64_t gsi = found_override ? override->gsi : irq;
+
+  uint64_t redirflags = 0;
+  redirflags |= (vec & 0xFF);
+  redirflags |= flags;
+  redirflags |= (lapic_id << 56);
 
   if (found_override) {
-    uint32_t polarity = ((override->flags & 0x03) == 0x03) ? 1 : 0;
-    uint32_t mode = (((override->flags >> 2) & 0x03) == 0x03) ? 1 : 0;
-    calc_flags |= (uint64_t)mode << 15;
-    calc_flags |= (uint64_t)polarity << 13;
+    switch (override->flags & 0x03) {
+    case 0:
+      break;
+    case 1:
+      redirflags &= ~(1 << 13);
+      break;
+    case 3:
+      redirflags |= (1 << 13);
+      break;
     }
 
-  uint32_t gsi = found_override ? override->gsi : irq;
+    switch ((override->flags >> 2) & 0x03) {
+    case 0:
+      break;
+    case 1:
+      redirflags &= ~(1 << 15);
+      break;
+    case 3:
+      redirflags |= (1 << 15);
+      break;
+    }
+  }
 
   ioapic = ioapic_find (gsi);
 
-  if (ioapic == NULL)
+  if (!ioapic)
     return;
 
   uint32_t irq_reg = ((gsi - ioapic->table_data.gsi_base) * 2) + 0x10;
-
-  ioapic_write (ioapic, irq_reg + 1, (uint32_t)(calc_flags >> 32));
-  ioapic_write (ioapic, irq_reg, (uint32_t)calc_flags);
+  ioapic_write (ioapic, irq_reg + 1, (uint32_t)(redirflags >> 32));
+  ioapic_write (ioapic, irq_reg, (uint32_t)redirflags);
 }
 
 /* Find and initialize the IOAPIC */
@@ -171,6 +190,9 @@ void ioapic_init (void) {
       struct acpi_madt_interrupt_source_override* override =
           (struct acpi_madt_interrupt_source_override*)current;
       intr_src_overrides[intr_src_override_entries++] = *override;
+
+      DEBUG ("Interrupt override: source=%u gsi=%u flags=%04x bus=%u\n", override->source,
+             override->gsi, override->flags, override->bus);
     } break;
     }
 

kernel/device/idedrv.c

@@ -41,13 +41,12 @@
 
 #define IDE_READ  1
 #define IDE_WRITE 2
-#define IDE_FLUSH 3
 
 static bool ide_wait (uint16_t io, uint32_t timeout, bool drq, bool errcheck) {
   uint32_t i = 0;
   uint8_t status;
 
-  for (;;) {
+  while (1) {
     status = inb (io + IDE_REG_STATUS);
 
     if (!(status & IDE_BSY))
@@ -57,28 +56,28 @@ static bool ide_wait (uint16_t io, uint32_t timeout, bool drq, bool errcheck) {
       return false;
   }
 
-  if (drq) {
-    i = 0;
-    while (!(status & IDE_DRQ) || (status & IDE_BSY)) {
-      if (status & IDE_ERR)
+  if (errcheck && (status & (IDE_ERR | IDE_DF)))
     return false;
 
+  if (!drq)
+    return true;
+
+  i = 0;
+
+  while (1) {
     status = inb (io + IDE_REG_STATUS);
 
-      if (status == 0xFF)
+    if (status & (IDE_ERR | IDE_DF))
       return false;
 
+    if (status & IDE_DRQ)
+      return true;
+
     if (++i >= timeout)
       return false;
   }
 }
 
-  if (errcheck && (status & (IDE_DF | IDE_ERR)))
-    return false;
-
-  return true;
-}
-
 #pragma clang optimize off
 static void ide_delay (uint16_t ctrl) {
   inb (ctrl);
@@ -95,54 +94,37 @@ static void ide_irq (void* arg, void* regs, bool user, struct reschedule_ctx* rc
 
   spin_lock (&idedrv->device->lock, &fd);
 
-  struct list_node_link* node = idedrv->requests;
+  struct idedrv_request* req = idedrv->current_req;
 
-  if (node == NULL) {
+  if (req == NULL) {
     (void)inb (idedrv->io + IDE_REG_STATUS);
     spin_unlock (&idedrv->device->lock, fd);
     return;
   }
 
-  struct idedrv_request* req = list_entry (node, struct idedrv_request, requests_link);
-
   uint8_t status = inb (idedrv->io + IDE_REG_STATUS);
 
-  if (req->type == IDE_FLUSH) {
-    list_remove (idedrv->requests, &req->requests_link);
-    spin_unlock (&idedrv->device->lock, fd);
-    atomic_store (&req->done, 1);
-    return;
-  }
-
   if ((status & (IDE_ERR | IDE_DF))) {
-    list_remove (idedrv->requests, &req->requests_link);
-    spin_unlock (&idedrv->device->lock, fd);
     atomic_store (&req->done, 1);
+    idedrv->current_req = NULL;
+    spin_unlock (&idedrv->device->lock, fd);
     return;
   }
 
-  if (!(status & IDE_BSY) && (status & IDE_DRQ)) {
-    switch (req->type) {
-    case IDE_READ: {
+  if ((status & IDE_DRQ) && (req->sector_done_count < req->sector_count)) {
     uint16_t* p = req->buffer + (req->sector_done_count * (idedrv->sector_size / 2));
+
+    if (req->type == IDE_READ)
       insw (idedrv->io + IDE_REG_DATA, p, idedrv->sector_size / 2);
-      req->sector_done_count++;
-    } break;
-    case IDE_WRITE: {
-      req->sector_done_count++;
-      if (req->sector_done_count < req->sector_count) {
-        uint16_t* p = req->buffer + (req->sector_done_count * (idedrv->sector_size / 2));
+    else
       outsw (idedrv->io + IDE_REG_DATA, p, idedrv->sector_size / 2);
-      }
-    } break;
-    }
+
+    req->sector_done_count++;
   }
 
-  if (req->sector_done_count >= req->sector_count) {
-    list_remove (idedrv->requests, &req->requests_link);
-    spin_unlock (&idedrv->device->lock, fd);
+  if ((req->sector_done_count >= req->sector_count)) {
     atomic_store (&req->done, 1);
-    return;
+    idedrv->current_req = NULL;
   }
 
   spin_unlock (&idedrv->device->lock, fd);
@@ -175,7 +157,7 @@ void ide_probe (uint16_t io, uint16_t ctrl, uint8_t devno, struct ide_probe* pro
   if (status == 0)
     return;
 
-  if (!ide_wait (io, 100000, true, true)) {
+  if (!ide_wait (io, 90000000, true, true)) {
     return;
   }
 
@@ -183,13 +165,11 @@ void ide_probe (uint16_t io, uint16_t ctrl, uint8_t devno, struct ide_probe* pro
 
   probe->flags |= IDE_PROBE_AVAIL;
 
-  if ((identify_buffer[106] & 0xC000) == 0x4000) {
-    if (identify_buffer[106] & (1 << 12)) {
+  if ((identify_buffer[106] & 0xC000) == 0x4000 && identify_buffer[106] & (1 << 12)) {
     uint32_t words_per_sector =
         (uint32_t)identify_buffer[117] | ((uint32_t)identify_buffer[118] << 16);
     probe->sector_size = (size_t)words_per_sector * 2;
   }
-  }
 
   if ((identify_buffer[83] & (1 << 10)) != 0)
     probe->flags |= IDE_PROBE_LBA48;
@@ -211,14 +191,21 @@ void ide_probe (uint16_t io, uint16_t ctrl, uint8_t devno, struct ide_probe* pro
     probe->sector_size = 512;
 }
 
-static void ide_prepare (struct idedrv* idedrv, size_t sector, uint16_t sector_count, bool clear) {
-  if (clear)
-    outb (idedrv->ctrl, 0x00);
+static void ide_prepare (struct idedrv* idedrv, size_t sector, uint16_t sector_count,
+                         bool irq_enable) {
+  uint8_t ctrl = inb (idedrv->ctrl);
+  if (irq_enable)
+    ctrl &= ~0x02;
+  else
+    ctrl |= 0x02;
+  outb (idedrv->ctrl, ctrl);
 
   if (idedrv->lba48) {
     outb (idedrv->io + IDE_REG_DRIVE, 0x40 | (idedrv->devno << 4));
     ide_delay (idedrv->ctrl);
 
+    ide_wait (idedrv->io, 100000, false, false);
+
     outb (idedrv->io + IDE_REG_SECCOUNT, (sector_count >> 8) & 0xFF);
     outb (idedrv->io + IDE_REG_LBA0, (sector >> 24) & 0xFF);
     outb (idedrv->io + IDE_REG_LBA1, (sector >> 32) & 0xFF);
@@ -229,9 +216,11 @@ static void ide_prepare (struct idedrv* idedrv, size_t sector, uint16_t sector_c
     outb (idedrv->io + IDE_REG_LBA1, (sector >> 8) & 0xFF);
     outb (idedrv->io + IDE_REG_LBA2, (sector >> 16) & 0xFF);
   } else {
-    outb (idedrv->io + IDE_REG_DRIVE, 0xE0 | (idedrv->devno << 4) | ((sector >> 24) & 0xFF));
+    outb (idedrv->io + IDE_REG_DRIVE, 0xE0 | (idedrv->devno << 4) | ((sector >> 24) & 0x0F));
     ide_delay (idedrv->ctrl);
 
+    ide_wait (idedrv->io, 100000, false, false);
+
     uint8_t count = (sector_count == 256) ? 0 : (uint8_t)sector_count;
     outb (idedrv->io + IDE_REG_SECCOUNT, count);
     outb (idedrv->io + IDE_REG_LBA0, sector & 0xFF);
@@ -258,17 +247,14 @@ bool idedrv_init (struct device* device, void* arg, struct proc* proc,
   idedrv->io = init->io;
   idedrv->ctrl = init->ctrl;
   idedrv->devno = init->devno;
-  idedrv->primscnd = init->primscnd;
+  idedrv->irq = init->irq;
+  idedrv->current_req = NULL;
+  idedrv->irqs_support = init->irqs_support;
 
   device->udata = idedrv;
 
-  if (idedrv->primscnd == 1) {
-    ioapic_route_irq (IDE_DRIVE_PRIM, 14, 0, thiscpu->lapic_id);
-    irq_attach (&ide_irq, idedrv, IDE_DRIVE_PRIM);
-  } else if (idedrv->primscnd == 2) {
-    ioapic_route_irq (IDE_DRIVE_SCND, 15, 0, thiscpu->lapic_id);
-    irq_attach (&ide_irq, idedrv, IDE_DRIVE_SCND);
-  }
+  if (idedrv->irqs_support)
+    irq_attach (&ide_irq, idedrv, idedrv->irq);
 
   return true;
 }
@@ -276,14 +262,14 @@ bool idedrv_init (struct device* device, void* arg, struct proc* proc,
 void idedrv_fini (struct device* device, struct proc* proc, struct reschedule_ctx* rctx) {
   struct idedrv* idedrv = device->udata;
 
-  struct list_node_link *req_link, *tmp_req_link;
-  list_foreach (idedrv->requests, req_link, tmp_req_link) {
-    struct idedrv_request* req = list_entry (req_link, struct idedrv_request, requests_link);
-    list_remove (idedrv->requests, &req->requests_link);
-    free (req);
+  if (idedrv->current_req != NULL) {
+    free (idedrv->current_req);
+    idedrv->current_req = NULL;
   }
 
-  irq_detach (idedrv->primscnd == 1 ? IDE_DRIVE_PRIM : IDE_DRIVE_SCND);
+  if (idedrv->irqs_support)
+    irq_detach (idedrv->irq);
+
   free (idedrv);
 }
 
@@ -303,6 +289,10 @@ int idedrv_read (struct device* device, struct proc* proc, struct reschedule_ctx
   if (sector + sector_count > idedrv->sector_count)
     return -ST_OOB_ERROR;
 
+  if (!ide_wait (idedrv->io, 100000, false, false))
+    return -ST_XDRV_READ_ERROR;
+
+  if (idedrv->irqs_support) {
     struct idedrv_request* req = malloc (sizeof (*req));
 
     if (req == NULL)
@@ -314,23 +304,13 @@ int idedrv_read (struct device* device, struct proc* proc, struct reschedule_ctx
     req->sector_done_count = 0;
     req->type = IDE_READ;
 
-  list_append (idedrv->requests, &req->requests_link);
+    idedrv->current_req = req;
 
     ide_prepare (idedrv, sector, sector_count, true);
 
     uint8_t cmd = idedrv->lba48 ? IDE_CMD_READ48 : IDE_CMD_READ28;
     outb (idedrv->io + IDE_REG_CMD, cmd);
 
-  if (!ide_wait (idedrv->io, 100000, true, true)) {
-    list_remove (idedrv->requests, &req->requests_link);
-    free (req);
-    return -ST_XDRV_READ_ERROR;
-  }
-
-  insw (idedrv->io + IDE_REG_DATA, buffer, idedrv->sector_size / 2);
-
-  req->sector_done_count = 1;
-
     spin_unlock (&device->lock, *lockflags);
 
     while (!atomic_load (&req->done))
@@ -339,6 +319,20 @@ int idedrv_read (struct device* device, struct proc* proc, struct reschedule_ctx
     spin_lock (&device->lock, lockflags);
 
     free (req);
+  } else {
+    ide_prepare (idedrv, sector, sector_count, false);
+
+    uint8_t cmd = idedrv->lba48 ? IDE_CMD_READ48 : IDE_CMD_READ28;
+    outb (idedrv->io + IDE_REG_CMD, cmd);
+
+    for (uint16_t s = 0; s < sector_count; s++) {
+      if (!ide_wait (idedrv->io, 100000, true, true))
+        return -ST_XDRV_READ_ERROR;
+
+      insw (idedrv->io + IDE_REG_DATA, buffer + (s * (idedrv->sector_size / 2)),
+            idedrv->sector_size / 2);
+    }
+  }
 
   return ST_OK;
 }
@@ -360,33 +354,22 @@ int idedrv_write (struct device* device, struct proc* proc, struct reschedule_ct
   if (sector + sector_count > idedrv->sector_count)
     return -ST_OOB_ERROR;
 
-  struct idedrv_request* req = malloc (sizeof (*req));
+  if (!ide_wait (idedrv->io, 100000, false, false))
+    return -ST_XDRV_WRITE_ERROR;
 
-  struct idedrv_request* flushreq = malloc (sizeof (*req));
+  if (idedrv->irqs_support) {
+    struct idedrv_request* req = malloc (sizeof (*req));
 
     if (req == NULL)
       return -ST_OOM_ERROR;
 
-  if (flushreq == NULL) {
-    free (req);
-    return -ST_OOM_ERROR;
-  }
-
     memset (req, 0, sizeof (*req));
     req->buffer = buffer;
     req->sector_count = sector_count;
     req->sector_done_count = 0;
     req->type = IDE_WRITE;
 
-  list_append (idedrv->requests, &req->requests_link);
-
-  memset (flushreq, 0, sizeof (*flushreq));
-  flushreq->buffer = NULL;
-  flushreq->sector_count = 0;
-  flushreq->sector_done_count = 0;
-  flushreq->type = IDE_FLUSH;
-
-  list_append (idedrv->requests, &flushreq->requests_link);
+    idedrv->current_req = req;
 
     ide_prepare (idedrv, sector, sector_count, true);
 
@@ -394,7 +377,7 @@ int idedrv_write (struct device* device, struct proc* proc, struct reschedule_ct
     outb (idedrv->io + IDE_REG_CMD, cmd);
 
     if (!ide_wait (idedrv->io, 100000, true, true)) {
-    list_remove (idedrv->requests, &req->requests_link);
+      idedrv->current_req = NULL;
       free (req);
       return -ST_XDRV_WRITE_ERROR;
     }
@@ -410,20 +393,38 @@ int idedrv_write (struct device* device, struct proc* proc, struct reschedule_ct
 
     spin_lock (&device->lock, lockflags);
 
+    free (req);
+  } else {
+    ide_prepare (idedrv, sector, sector_count, false);
+
+    uint8_t cmd = idedrv->lba48 ? IDE_CMD_WRITE48 : IDE_CMD_WRITE28;
+    outb (idedrv->io + IDE_REG_CMD, cmd);
+
+    for (uint16_t s = 0; s < sector_count; s++) {
+      if (!ide_wait (idedrv->io, 100000, true, true))
+        return -ST_XDRV_WRITE_ERROR;
+
+      outsw (idedrv->io + IDE_REG_DATA, buffer + (s * (idedrv->sector_size / 2)),
+             idedrv->sector_size / 2);
+    }
+  }
+
+  uint8_t ctrl = inb (idedrv->ctrl);
+  ctrl |= 0x02;
+  outb (idedrv->ctrl, ctrl);
+
   if (idedrv->lba48)
     outb (idedrv->io + IDE_REG_CMD, IDE_CMD_FLUSH48);
   else
     outb (idedrv->io + IDE_REG_CMD, IDE_CMD_FLUSH28);
 
-  spin_unlock (&device->lock, *lockflags);
+  uint8_t status;
+  do {
+    status = inb (idedrv->io + IDE_REG_STATUS);
+  } while (status & IDE_BSY);
 
-  while (!atomic_load (&flushreq->done))
-    spin_lock_relax ();
-
-  spin_lock (&device->lock, lockflags);
-
-  free (req);
-  free (flushreq);
+  if (status & (IDE_ERR | IDE_DF))
+    return -ST_XDRV_WRITE_ERROR;
 
   return ST_OK;
 }

kernel/device/idedrv.h

@@ -19,14 +19,14 @@ struct idedrv_init {
   size_t sector_size;
   uint16_t io, ctrl;
   uint8_t devno;
-  uint8_t primscnd;
+  uint8_t irq;
+  bool irqs_support;
 };
 
 struct idedrv_request {
   uint16_t* buffer;
   size_t sector_done_count;
   size_t sector_count;
-  struct list_node_link requests_link;
   int type;
   atomic_int done;
 };
@@ -38,9 +38,9 @@ struct idedrv {
   size_t sector_size;
   uint16_t io, ctrl;
   uint8_t devno;
-  uint8_t primscnd;
-
-  struct list_node_link* requests;
+  uint8_t irq;
+  struct idedrv_request* current_req;
+  bool irqs_support;
 };
 
 struct ide_probe {
@@ -49,7 +49,8 @@ struct ide_probe {
   size_t sector_size;
   uint16_t io, ctrl;
   uint8_t devno;
-  uint8_t primscnd;
+  uint8_t irq;
+  bool irqs_support;
 };
 
 bool idedrv_init (struct device* device, void* arg, struct proc* proc, struct reschedule_ctx* rctx);

kernel/device/pci_ide.c

@@ -1,3 +1,5 @@
+#include <amd64/apic.h>
+#include <amd64/intr_defs.h>
 #include <amd64/io.h>
 #include <device/device.h>
 #include <device/idedrv.h>
@@ -13,7 +15,8 @@
 #include <proc/reschedule.h>
 #include <sys/debug.h>
 
-static atomic_int ide_counter = 0;
+static int ide_counter = 0;
+/* static bool ide_legacy_claimed = false; */
 
 static const char* progif_msg[] = {
     [0x00] = "ISA Compatibility mode-only controller",
@@ -36,7 +39,7 @@ static void ide_make_device (struct proc* proc, struct reschedule_ctx* rctx,
          probe.devno, probe.sector_size, probe.sector_count);
 
   char device_key[fieldsizeof (struct device, key)];
-  snprintf (device_key, sizeof (device_key), "IDE%d", atomic_fetch_add (&ide_counter, 1));
+  snprintf (device_key, sizeof (device_key), "IDE%d", ide_counter++);
 
   device_op_func_t ops[] = {
       [XDRV_GET_SIZE] = &idedrv_get_size,
@@ -53,7 +56,8 @@ static void ide_make_device (struct proc* proc, struct reschedule_ctx* rctx,
       .io = probe.io,
       .ctrl = probe.ctrl,
       .devno = probe.devno,
-      .primscnd = probe.primscnd,
+      .irq = probe.irq,
+      .irqs_support = probe.irqs_support,
   };
   struct device* ide = device_create (device_key, ops, lengthof (ops), &idedrv_init, &idedrv_fini,
                                       &init, proc, rctx);
@@ -63,7 +67,7 @@ static void ide_make_device (struct proc* proc, struct reschedule_ctx* rctx,
 bool pci_ide_init (struct proc* proc, struct reschedule_ctx* rctx, struct pci_info pci_info) {
   uint16_t pci_cmd = pci_read16 (pci_info.bus, pci_info.slot, pci_info.func, PCI_COMMAND);
 
-  uint16_t new_cmd = (pci_cmd | (1 << 0)) & ~(1 << 10);
+  uint16_t new_cmd = (pci_cmd | (1 << 0) | (1 << 2)) & ~(1 << 10);
 
   if (pci_cmd != new_cmd) {
     pci_write16 (pci_info.bus, pci_info.slot, pci_info.func, PCI_COMMAND, new_cmd);
@@ -114,20 +118,35 @@ bool pci_ide_init (struct proc* proc, struct reschedule_ctx* rctx, struct pci_in
     sctrl = 0x376;
   }
 
-  uint16_t channels[2][2] = {{pcmd, pctrl}, {scmd, sctrl}};
+  bool irqs_support = false;
+
+  if ((progif & 0x05)) {
+    irqs_support = false;
+  } else {
+    irqs_support = true;
+    ioapic_route_irq (IDE_DRIVE_PRIM, 14, 0, thiscpu->lapic_id);
+    ioapic_route_irq (IDE_DRIVE_SCND, 15, 0, thiscpu->lapic_id);
+  }
+
+  DEBUG ("pcmd=%x, pctrl=%x\n", pcmd, pctrl);
+  DEBUG ("scmd=%x, sctrl=%x\n", scmd, sctrl);
+  DEBUG ("IRQ support=%d\n", irqs_support);
+
+  uint16_t channels[2][3] = {{pcmd, pctrl, IDE_DRIVE_PRIM}, {scmd, sctrl, IDE_DRIVE_SCND}};
 
   for (size_t i = 0; i < lengthof (channels); i++) {
     uint16_t cmd = channels[i][0];
     uint16_t ctrl = channels[i][1];
-
-    if (cmd == 0)
-      continue;
-
-    probe.primscnd = i + 1;
+    uint8_t irq = channels[i][2];
 
     for (size_t dev = 0; dev < 2; dev++) {
       ide_probe (cmd, ctrl, dev, &probe);
 
+      probe.ctrl = ctrl;
+      probe.io = cmd;
+      probe.irq = irq;
+      probe.irqs_support = irqs_support;
+
       if ((probe.flags & IDE_PROBE_AVAIL))
         ide_make_device (proc, rctx, probe);
     }