433 lines
9.7 KiB
C
433 lines
9.7 KiB
C
#ifndef SYS__CPU_H__
|
|
#define SYS__CPU_H__
|
|
|
|
#include <stdint.h>
|
|
#include <stddef.h>
|
|
#include <stdbool.h>
|
|
|
|
#if defined(__x86_64__) || defined(__i386__)
|
|
|
|
static inline bool cpuid(uint32_t leaf, uint32_t subleaf,
|
|
uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) {
|
|
uint32_t cpuid_max;
|
|
asm volatile ("cpuid"
|
|
: "=a" (cpuid_max)
|
|
: "a" (leaf & 0x80000000)
|
|
: "ebx", "ecx", "edx");
|
|
if (leaf > cpuid_max)
|
|
return false;
|
|
asm volatile ("cpuid"
|
|
: "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx)
|
|
: "a" (leaf), "c" (subleaf));
|
|
return true;
|
|
}
|
|
|
|
static inline void outb(uint16_t port, uint8_t value) {
|
|
asm volatile ("outb %%al, %1" : : "a" (value), "Nd" (port) : "memory");
|
|
}
|
|
|
|
static inline void outw(uint16_t port, uint16_t value) {
|
|
asm volatile ("outw %%ax, %1" : : "a" (value), "Nd" (port) : "memory");
|
|
}
|
|
|
|
static inline void outd(uint16_t port, uint32_t value) {
|
|
asm volatile ("outl %%eax, %1" : : "a" (value), "Nd" (port) : "memory");
|
|
}
|
|
|
|
static inline uint8_t inb(uint16_t port) {
|
|
uint8_t value;
|
|
asm volatile ("inb %1, %%al" : "=a" (value) : "Nd" (port) : "memory");
|
|
return value;
|
|
}
|
|
|
|
static inline uint16_t inw(uint16_t port) {
|
|
uint16_t value;
|
|
asm volatile ("inw %1, %%ax" : "=a" (value) : "Nd" (port) : "memory");
|
|
return value;
|
|
}
|
|
|
|
static inline uint32_t ind(uint16_t port) {
|
|
uint32_t value;
|
|
asm volatile ("inl %1, %%eax" : "=a" (value) : "Nd" (port) : "memory");
|
|
return value;
|
|
}
|
|
|
|
static inline void mmoutb(uintptr_t addr, uint8_t value) {
|
|
asm volatile (
|
|
"movb %1, (%0)"
|
|
:
|
|
: "r" (addr), "ir" (value)
|
|
: "memory"
|
|
);
|
|
}
|
|
|
|
static inline void mmoutw(uintptr_t addr, uint16_t value) {
|
|
asm volatile (
|
|
"movw %1, (%0)"
|
|
:
|
|
: "r" (addr), "ir" (value)
|
|
: "memory"
|
|
);
|
|
}
|
|
|
|
static inline void mmoutd(uintptr_t addr, uint32_t value) {
|
|
asm volatile (
|
|
"movl %1, (%0)"
|
|
:
|
|
: "r" (addr), "ir" (value)
|
|
: "memory"
|
|
);
|
|
}
|
|
|
|
#if defined (__x86_64__)
|
|
static inline void mmoutq(uintptr_t addr, uint64_t value) {
|
|
asm volatile (
|
|
"movq %1, (%0)"
|
|
:
|
|
: "r" (addr), "r" (value)
|
|
: "memory"
|
|
);
|
|
}
|
|
#endif
|
|
|
|
static inline uint8_t mminb(uintptr_t addr) {
|
|
uint8_t ret;
|
|
asm volatile (
|
|
"movb (%1), %0"
|
|
: "=r" (ret)
|
|
: "r" (addr)
|
|
: "memory"
|
|
);
|
|
return ret;
|
|
}
|
|
|
|
static inline uint16_t mminw(uintptr_t addr) {
|
|
uint16_t ret;
|
|
asm volatile (
|
|
"movw (%1), %0"
|
|
: "=r" (ret)
|
|
: "r" (addr)
|
|
: "memory"
|
|
);
|
|
return ret;
|
|
}
|
|
|
|
static inline uint32_t mmind(uintptr_t addr) {
|
|
uint32_t ret;
|
|
asm volatile (
|
|
"movl (%1), %0"
|
|
: "=r" (ret)
|
|
: "r" (addr)
|
|
: "memory"
|
|
);
|
|
return ret;
|
|
}
|
|
|
|
#if defined (__x86_64__)
|
|
static inline uint64_t mminq(uintptr_t addr) {
|
|
uint64_t ret;
|
|
asm volatile (
|
|
"movq (%1), %0"
|
|
: "=r" (ret)
|
|
: "r" (addr)
|
|
: "memory"
|
|
);
|
|
return ret;
|
|
}
|
|
#endif
|
|
|
|
static inline uint64_t rdmsr(uint32_t msr) {
|
|
uint32_t edx, eax;
|
|
asm volatile ("rdmsr"
|
|
: "=a" (eax), "=d" (edx)
|
|
: "c" (msr)
|
|
: "memory");
|
|
return ((uint64_t)edx << 32) | eax;
|
|
}
|
|
|
|
static inline void wrmsr(uint32_t msr, uint64_t value) {
|
|
uint32_t edx = value >> 32;
|
|
uint32_t eax = (uint32_t)value;
|
|
asm volatile ("wrmsr"
|
|
:
|
|
: "a" (eax), "d" (edx), "c" (msr)
|
|
: "memory");
|
|
}
|
|
|
|
static inline uint64_t rdtsc(void) {
|
|
uint32_t edx, eax;
|
|
asm volatile ("rdtsc" : "=a" (eax), "=d" (edx) :: "memory");
|
|
return ((uint64_t)edx << 32) | eax;
|
|
}
|
|
|
|
static inline uint64_t tsc_freq_arch(void) {
|
|
uint32_t eax, ebx, ecx, edx;
|
|
if (!cpuid(0x15, 0, &eax, &ebx, &ecx, &edx))
|
|
return 0;
|
|
if (eax == 0 || ebx == 0 || ecx == 0)
|
|
return 0;
|
|
return (uint64_t)ecx * ebx / eax;
|
|
}
|
|
|
|
#define rdrand(type) ({ \
|
|
type rdrand__ret; \
|
|
asm volatile ( \
|
|
"1: " \
|
|
"rdrand %0;" \
|
|
"jnc 1b;" \
|
|
: "=r" (rdrand__ret) \
|
|
); \
|
|
rdrand__ret; \
|
|
})
|
|
|
|
#define rdseed(type) ({ \
|
|
type rdseed__ret; \
|
|
asm volatile ( \
|
|
"1: " \
|
|
"rdseed %0;" \
|
|
"jnc 1b;" \
|
|
: "=r" (rdseed__ret) \
|
|
); \
|
|
rdseed__ret; \
|
|
})
|
|
|
|
#define write_cr(reg, val) do { \
|
|
asm volatile ("mov %0, %%cr" reg :: "r" (val) : "memory"); \
|
|
} while (0)
|
|
|
|
#define read_cr(reg) ({ \
|
|
size_t read_cr__cr; \
|
|
asm volatile ("mov %%cr" reg ", %0" : "=r" (read_cr__cr) :: "memory"); \
|
|
read_cr__cr; \
|
|
})
|
|
|
|
#define locked_read(var) ({ \
|
|
typeof(*var) locked_read__ret = 0; \
|
|
asm volatile ( \
|
|
"lock xadd %0, %1" \
|
|
: "+r" (locked_read__ret) \
|
|
: "m" (*(var)) \
|
|
: "memory" \
|
|
); \
|
|
locked_read__ret; \
|
|
})
|
|
|
|
#define locked_write(var, val) do { \
|
|
__auto_type locked_write__ret = val; \
|
|
asm volatile ( \
|
|
"lock xchg %0, %1" \
|
|
: "+r" ((locked_write__ret)) \
|
|
: "m" (*(var)) \
|
|
: "memory" \
|
|
); \
|
|
} while (0)
|
|
|
|
#elif defined (__aarch64__)
|
|
|
|
static inline uint64_t rdtsc(void) {
|
|
uint64_t v;
|
|
asm volatile ("mrs %0, cntpct_el0" : "=r" (v));
|
|
return v;
|
|
}
|
|
|
|
static inline uint64_t tsc_freq_arch(void) {
|
|
uint64_t v;
|
|
asm volatile ("mrs %0, cntfrq_el0" : "=r" (v));
|
|
return v;
|
|
}
|
|
|
|
#define locked_read(var) ({ \
|
|
typeof(*var) locked_read__ret = 0; \
|
|
asm volatile ( \
|
|
"ldar %0, %1" \
|
|
: "=r" (locked_read__ret) \
|
|
: "m" (*(var)) \
|
|
: "memory" \
|
|
); \
|
|
locked_read__ret; \
|
|
})
|
|
|
|
static inline size_t icache_line_size(void) {
|
|
uint64_t ctr;
|
|
asm volatile ("mrs %0, ctr_el0" : "=r"(ctr));
|
|
|
|
return 4 << (ctr & 0b1111);
|
|
}
|
|
|
|
static inline size_t dcache_line_size(void) {
|
|
uint64_t ctr;
|
|
asm volatile ("mrs %0, ctr_el0" : "=r"(ctr));
|
|
|
|
return 4 << ((ctr >> 16) & 0b1111);
|
|
}
|
|
|
|
static inline bool is_icache_pipt(void) {
|
|
uint64_t ctr;
|
|
asm volatile ("mrs %0, ctr_el0" : "=r"(ctr));
|
|
|
|
return ((ctr >> 14) & 0b11) == 0b11;
|
|
}
|
|
|
|
// Clean D-Cache to Point of Coherency
|
|
static inline void clean_dcache_poc(uintptr_t start, uintptr_t end) {
|
|
size_t dsz = dcache_line_size();
|
|
|
|
uintptr_t addr = start & ~(dsz - 1);
|
|
while (addr < end) {
|
|
asm volatile ("dc cvac, %0" :: "r"(addr) : "memory");
|
|
addr += dsz;
|
|
}
|
|
|
|
asm volatile ("dsb sy\n\tisb");
|
|
}
|
|
|
|
// Invalidate I-Cache to Point of Unification
|
|
static inline void inval_icache_pou(uintptr_t start, uintptr_t end) {
|
|
if (!is_icache_pipt()) {
|
|
asm volatile ("ic ialluis" ::: "memory");
|
|
asm volatile ("dsb sy\n\tisb");
|
|
return;
|
|
}
|
|
|
|
size_t isz = icache_line_size();
|
|
|
|
uintptr_t addr = start & ~(isz - 1);
|
|
while (addr < end) {
|
|
asm volatile ("ic ivau, %0" :: "r"(addr) : "memory");
|
|
addr += isz;
|
|
}
|
|
|
|
asm volatile ("dsb sy\n\tisb");
|
|
}
|
|
|
|
static inline int current_el(void) {
|
|
uint64_t v;
|
|
|
|
asm volatile ("mrs %0, currentel" : "=r"(v));
|
|
v = (v >> 2) & 0b11;
|
|
|
|
return v;
|
|
}
|
|
|
|
#elif defined (__riscv)
|
|
|
|
static inline uint64_t rdtsc(void) {
|
|
uint64_t v;
|
|
asm volatile ("rdtime %0" : "=r"(v));
|
|
return v;
|
|
}
|
|
|
|
uint64_t riscv_time_base_frequency(void);
|
|
|
|
static inline uint64_t tsc_freq_arch(void) {
|
|
return riscv_time_base_frequency();
|
|
}
|
|
|
|
#define csr_read(csr) ({\
|
|
size_t v;\
|
|
asm volatile ("csrr %0, " csr : "=r"(v));\
|
|
v;\
|
|
})
|
|
|
|
#define csr_write(csr, v) ({\
|
|
size_t old;\
|
|
asm volatile ("csrrw %0, " csr ", %1" : "=r"(old) : "r"(v));\
|
|
old;\
|
|
})
|
|
|
|
#define make_satp(mode, ppn) (((size_t)(mode) << 60) | ((size_t)(ppn) >> 12))
|
|
|
|
#define locked_read(var) ({ \
|
|
typeof(*var) locked_read__ret; \
|
|
asm volatile ( \
|
|
"ld %0, (%1); fence r, rw" \
|
|
: "=r"(locked_read__ret) \
|
|
: "r"(var) \
|
|
: "memory" \
|
|
); \
|
|
locked_read__ret; \
|
|
})
|
|
|
|
extern size_t bsp_hartid;
|
|
|
|
struct riscv_hart {
|
|
struct riscv_hart *next;
|
|
const char *isa_string;
|
|
size_t hartid;
|
|
uint32_t acpi_uid;
|
|
uint8_t mmu_type;
|
|
uint8_t flags;
|
|
};
|
|
|
|
#define RISCV_HART_COPROC ((uint8_t)1 << 0) // is a coprocessor
|
|
#define RISCV_HART_HAS_MMU ((uint8_t)1 << 1) // `mmu_type` field is valid
|
|
|
|
extern struct riscv_hart *hart_list;
|
|
extern struct riscv_hart *bsp_hart;
|
|
|
|
bool riscv_check_isa_extension_for(size_t hartid, const char *ext, size_t *maj, size_t *min);
|
|
|
|
static inline bool riscv_check_isa_extension(const char *ext, size_t *maj, size_t *min) {
|
|
return riscv_check_isa_extension_for(bsp_hartid, ext, maj, min);
|
|
}
|
|
|
|
void init_riscv(const char *config);
|
|
|
|
#elif defined (__loongarch64)
|
|
|
|
static inline uint64_t rdtsc(void) {
|
|
uint64_t v;
|
|
asm volatile ("rdtime.d %0, $zero" : "=r" (v));
|
|
return v;
|
|
}
|
|
|
|
static inline uint32_t loongarch_cpucfg(uint32_t reg) {
|
|
uint32_t v;
|
|
asm volatile ("cpucfg %0, %1" : "=r" (v) : "r" (reg));
|
|
return v;
|
|
}
|
|
|
|
static inline uint64_t tsc_freq_arch(void) {
|
|
uint32_t cc_freq = loongarch_cpucfg(4);
|
|
uint32_t cc_cfg = loongarch_cpucfg(5);
|
|
uint32_t cc_mul = cc_cfg & 0xFFFF;
|
|
uint32_t cc_div = (cc_cfg >> 16) & 0xFFFF;
|
|
if (cc_freq == 0 || cc_mul == 0 || cc_div == 0) {
|
|
return 0;
|
|
}
|
|
return (uint64_t)cc_freq * cc_mul / cc_div;
|
|
}
|
|
|
|
#else
|
|
#error Unknown architecture
|
|
#endif
|
|
|
|
extern uint64_t tsc_freq;
|
|
void calibrate_tsc(void);
|
|
|
|
static inline uint64_t rdtsc_usec(void) {
|
|
uint64_t exec_ticks = rdtsc();
|
|
if (tsc_freq == 0) {
|
|
return 0;
|
|
}
|
|
return exec_ticks / tsc_freq * 1000000
|
|
+ exec_ticks % tsc_freq * 1000000 / tsc_freq;
|
|
}
|
|
|
|
static inline void stall(uint64_t us) {
|
|
#if defined(BIOS)
|
|
if (tsc_freq == 0) {
|
|
// ~1 us per inb on ISA/LPC bus
|
|
for (uint64_t i = 0; i < us; i++) {
|
|
inb(0x80);
|
|
}
|
|
return;
|
|
}
|
|
#endif
|
|
uint64_t ticks = (tsc_freq * us + 999999) / 1000000;
|
|
uint64_t next_stop = rdtsc() + ticks;
|
|
while (rdtsc() < next_stop);
|
|
}
|
|
|
|
#endif
|